distributed_strategy.py 89.7 KB
Newer Older
1 2
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
# Copyright (c) 2021 NVIDIA Corporation. All rights reserved.
3 4 5 6 7 8 9 10 11 12 13 14 15
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16 17
import copy

18 19 20
import google.protobuf
import google.protobuf.text_format

21
import paddle
22
from paddle.distributed.fleet.proto import distributed_strategy_pb2
23
from paddle.distributed.fleet.utils.log_util import logger
24
from paddle.fluid.framework import _global_flags
25
from paddle.fluid.wrapped_decorator import wrap_decorator
26

27
__all__ = []
28

29 30 31 32 33 34 35 36 37 38 39 40 41 42
non_auto_func_called = True


def __non_auto_func_called__(func):
    def __impl__(*args, **kwargs):
        global non_auto_func_called
        non_auto_func_called = False
        return func(*args, **kwargs)

    return __impl__


is_strict_auto = wrap_decorator(__non_auto_func_called__)

43

44 45 46 47 48 49 50 51 52 53 54 55 56
def get_msg_dict(msg):
    res_dict = {}
    fields = msg.DESCRIPTOR.fields
    for f in fields:
        res_dict[f.name] = getattr(msg, f.name)
    return res_dict


def assign_configs_value(msg, config):
    fields = msg.DESCRIPTOR.fields
    for key in config:
        for f in fields:
            if key == f.name:
57 58 59
                # LABEL_OPTIONAL = 1
                # LABEL_REPEATED = 3
                # LABEL_REQUIRED = 2
60
                if f.label == 3:
S
sneaxiy 已提交
61 62
                    if config[f.name] is not None:
                        getattr(msg, f.name).extend(config[f.name])
63 64 65 66 67 68 69
                elif f.label == 1 or f.label == 2:
                    setattr(msg, f.name, config[f.name])


def check_configs_key(msg, config, field_name):
    key_list = msg.DESCRIPTOR.fields_by_name.keys()
    for key in config:
70
        assert key in key_list, f"key:{key} not in {field_name}"
71 72


73
class DistributedJobInfo:
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
    """
    DistributedJobInfo will serialize all distributed training information
    Just for inner use: 1) debug 2) replicate experiments
    """

    def __init__(self):
        self.job_info = distributed_strategy_pb2.DistributedJobInfo()

    def _set_worker_num(self, worker_num):
        self.job_info.worker_num = worker_num

    def _set_server_num(self, server_num):
        self.job_info.server_num = server_num

    def _set_worker_ips(self, worker_ips):
        self.job_info.worker_ips.extend(worker_ips)

    def _set_server_endpoints(self, server_endpoints):
        self.job_info.server_endpoints.extend(server_endpoints)

    def _set_origin_startup(self, origin_startup_prog):
        self.job_info.origin_startup = str(origin_startup_prog)

    def _set_origin_main(self, origin_main_prog):
        self.job_info.origin_main = str(origin_main_prog)

    def _distributed_main(self, distributed_main_prog):
        self.job_info.distributed_main = str(distributed_main_prog)

    def _optimizer_name(self, optimizer_name):
        self.job_info.optimizer_name = optimizer_name

    def _set_distributed_strategy(self, dist_strategy):
        self.job_info.strategy = dist_strategy


110 111 112
ReduceStrategyFleet = int


113
class DistributedStrategy:
114 115
    __lock_attr = False

116
    def __init__(self):
117
        """
118

119 120
        DistributedStrategy is the main configuration entry for distributed training of Paddle.
        All of the distributed training configurations can be configured in DistributedStrategy,
121
        such as automatic mixed precision (AMP), Layer-wise Adaptive Rate Scaling (LARS),
122
        asynchronous update parameter server(ASGD), etc.
1
123malin 已提交
123

124 125
        DistributedStrategy can be serialized into protobuf file or deserialized from protobuf file

126
        Users who run local training usually configure BuildStrategy and ExecutionStrategy, and
127 128 129
        DistributedStrategy supports configurations from BuildStrategy and ExecutionStrategy

        """
130
        self.strategy = distributed_strategy_pb2.DistributedStrategy()
131 132 133

        # Set the default values of the following flags to the ones set by users
        key = 'FLAGS_cudnn_batchnorm_spatial_persistent'
134
        if _global_flags().is_public(key):
135
            self.strategy.cudnn_batchnorm_spatial_persistent = bool(
136 137
                _global_flags()[key]
            )
138
        key = 'FLAGS_conv_workspace_size_limit'
139 140
        if _global_flags().is_public(key):
            self.strategy.conv_workspace_size_limit = int(_global_flags()[key])
141
        key = 'FLAGS_cudnn_exhaustive_search'
142 143
        if _global_flags().is_public(key):
            self.strategy.cudnn_exhaustive_search = bool(_global_flags()[key])
144
        key = 'FLAGS_sync_nccl_allreduce'
145 146
        if _global_flags().is_public(key):
            self.strategy.sync_nccl_allreduce = bool(_global_flags()[key])
147

148
        self.hybrid_parallel_order = ['dp', 'pp', 'sharding', 'mp']
149
        self.__lock_attr = True
150
        logger.info("distributed strategy initialized")
151 152 153

    def __setattr__(self, key, value):
        if self.__lock_attr and not hasattr(self, key):
154
            raise TypeError(
155
                f"{key} is not a attribute of {self.__class__.__name__}"
156
            )
157
        object.__setattr__(self, key, value)
158

159
    def save_to_prototxt(self, output):
160
        """
161

162 163 164
        Serialize current DistributedStrategy to string and save to output file

        Examples:
165
            .. code-block:: python
1
123malin 已提交
166

167 168 169 170 171 172
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.dgc = True
                strategy.recompute = True
                strategy.recompute_configs = {"checkpoints": ["x"]}
                strategy.save_to_prototxt("dist_strategy.prototxt")
1
123malin 已提交
173

174
        """
175 176 177 178
        with open(output, "w") as fout:
            fout.write(str(self.strategy))

    def load_from_prototxt(self, pb_file):
179
        """
180

181 182 183
        Load from prototxt file for DistributedStrategy initialization

        Examples:
184
            .. code-block:: python
1
123malin 已提交
185

186 187 188
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.load_from_prototxt("dist_strategy.prototxt")
189 190 191 192

        """
        with open(pb_file, 'r') as f:
            self.strategy = google.protobuf.text_format.Merge(
193 194
                str(f.read()), self.strategy
            )
195 196 197 198 199 200 201

    @property
    def execution_strategy(self):
        """
        Configure ExecutionStrategy for DistributedStrategy

        Examples:
202
            .. code-block:: python
1
123malin 已提交
203

204 205 206 207 208
                import paddle
                exe_strategy = paddle.static.ExecutionStrategy()
                exe_strategy.num_threads = 10
                exe_strategy.num_iteration_per_drop_scope = 10
                exe_strategy.num_iteration_per_run = 10
209

210 211
                strategy = paddle.distributed.fleet.DistributedStrategy()
                strategy.execution_strategy = exe_strategy
212 213

        """
W
wangxiaoning 已提交
214
        execution_strategy = paddle.static.ExecutionStrategy()
215 216
        fields = self.strategy.execution_strategy.DESCRIPTOR.fields
        for f in fields:
217 218 219 220 221
            setattr(
                execution_strategy,
                f.name,
                getattr(self.strategy.execution_strategy, f.name),
            )
222 223 224
        return execution_strategy

    @execution_strategy.setter
225
    @is_strict_auto
226 227 228
    def execution_strategy(self, strategy):
        fields = self.strategy.execution_strategy.DESCRIPTOR.fields
        for f in fields:
229 230 231 232 233
            setattr(
                self.strategy.execution_strategy,
                f.name,
                getattr(strategy, f.name),
            )
234 235 236 237

    @property
    def build_strategy(self):
        """
238

239 240 241 242 243
        Configure BuildStrategy for DistributedStrategy
        Note that the properties of BuildStrategy are valid in DistributedStrategy
        only if the property is non-distributed strategy.

        Examples:
244
            .. code-block:: python
1
123malin 已提交
245

246 247 248 249 250 251 252 253 254 255
                import paddle
                build_strategy = paddle.static.BuildStrategy()
                build_strategy.enable_sequential_execution = True
                build_strategy.fuse_elewise_add_act_ops = True
                build_strategy.fuse_bn_act_ops = True
                build_strategy.enable_auto_fusion = True
                build_strategy.fuse_relu_depthwise_conv = True
                build_strategy.fuse_broadcast_ops = True
                build_strategy.fuse_all_optimizer_ops = True
                build_strategy.enable_inplace = True
256

257 258
                strategy = paddle.distributed.fleet.DistributedStrategy()
                strategy.build_strategy = build_strategy
1
123malin 已提交
259

260 261
        """

W
wangxiaoning 已提交
262
        build_strategy = paddle.static.BuildStrategy()
263 264
        fields = self.strategy.build_strategy.DESCRIPTOR.fields
        for f in fields:
265 266
            value = getattr(self.strategy.build_strategy, f.name)
            if f.name == 'reduce_strategy':
267
                value = paddle.static.BuildStrategy.ReduceStrategy(value)
268
            setattr(build_strategy, f.name, value)
269 270 271
        return build_strategy

    @build_strategy.setter
272
    @is_strict_auto
273 274 275 276
    def build_strategy(self, strategy):
        fields = self.strategy.build_strategy.DESCRIPTOR.fields
        for f in fields:
            if f.label == 1 or f.label == 2:  # optional and required field
277 278 279 280
                value = getattr(strategy, f.name)
                if f.name == 'reduce_strategy':
                    value = ReduceStrategyFleet(value)
                setattr(self.strategy.build_strategy, f.name, value)
281
            elif f.label == 3:  # repeated field
282 283 284
                getattr(self.strategy.build_strategy, f.name).extend(
                    getattr(strategy, f.name)
                )
285 286

    @property
Y
Yuang Liu 已提交
287 288
    def gradient_scale_configs(self):
        """
289

Y
Yuang Liu 已提交
290
        Set the strategy of gradient scale
291

Y
Yuang Liu 已提交
292
        Examples:
293
            .. code-block:: python
Y
Yuang Liu 已提交
294

295 296 297
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.gradient_scale_configs = {'scale_strategy': 'avg'}
Y
Yuang Liu 已提交
298 299

        Note that, strategy must be in 'avg', 'sum' or 'customized'
300

Y
Yuang Liu 已提交
301 302 303 304 305 306
        """
        return get_msg_dict(self.strategy.gradient_scale_configs)

    @gradient_scale_configs.setter
    @is_strict_auto
    def gradient_scale_configs(self, config):
307 308 309 310 311
        check_configs_key(
            self.strategy.gradient_scale_configs,
            config,
            'gradient_scale_configs',
        )
Y
Yuang Liu 已提交
312 313 314
        assign_configs_value(self.strategy.gradient_scale_configs, config)

    @property
D
Dong Daxiang 已提交
315
    def a_sync(self):
316
        """
317

318
        Indicating whether we are using asynchronous stocastic gradient descent updates
319
        for training. This property is valid when we are using parameter server training,
320 321 322 323
        which is implied by setting approperate RoleMaker
        Default value: True

        Examples:
324
            .. code-block:: python
1
123malin 已提交
325

326 327 328
                import paddle.distributed.fleet as fleet
                role_maker = fleet.PaddleCloudRoleMaker()
                fleet.init(role_maker)
329

330 331
                strategy = fleet.DistributedStrategy()
                strategy.a_sync = True  # by default this is True
332

333 334
                # code block for defining loss and local optimizer
                # sgd = fleet.distributed_optimizer(optimizer, strategy)
1
123malin 已提交
335

336
        """
D
Dong Daxiang 已提交
337
        return self.strategy.a_sync
338

D
Dong Daxiang 已提交
339
    @a_sync.setter
340
    @is_strict_auto
D
Dong Daxiang 已提交
341
    def a_sync(self, flag):
342
        if isinstance(flag, bool):
D
Dong Daxiang 已提交
343
            self.strategy.a_sync = flag
344
            self.a_sync_configs = {"k_steps": 0}
345
        else:
346
            raise ValueError(
347 348 349 350
                "The type of `flag` is invalid, expected type is bool, but received {}".format(
                    type(flag)
                )
            )
351 352

    @property
D
Dong Daxiang 已提交
353
    def a_sync_configs(self):
354
        """
355

D
Dong Daxiang 已提交
356
        Set a_sync update configurations. In general, asynchronous parameter server
357 358
        training has serveral configurable settings that can be configured through
        a dict.
359

360
        **Notes**:
M
mapingshuo 已提交
361 362 363 364 365 366 367 368 369 370 371 372 373
            k_step(int): number of local optimization updates before communication

            max_merge_var_num(int): maximum number of merged gradients before communication

            send_queue_size(int): a buffer size of worker communication

            independent_recv_thread(bool): if we are using independent recv thread for communication

            thread_pool_size(int): number of thread pool

            send_wait_times(int): waiting time for sending gradients

            runtime_split_send_recv(bool): if we are using Tensor split for send and recv during runtime
374

375
        Examples:
376
            .. code-block:: python
1
123malin 已提交
377

378 379 380
                import paddle.distributed.fleet as fleet
                role_maker = fleet.PaddleCloudRoleMaker()
                fleet.init(role_maker)
381

382 383 384 385
                strategy = fleet.DistributedStrategy()
                strategy.a_sync = True  # by default this is True
                configs = {"k_steps": 1024, "send_queue_size": 32}
                strategy.a_sync_configs = configs
386

387 388
                # code block for defining loss and local optimizer
                # sgd = fleet.distributed_optimizer(optimizer, strategy)
M
mapingshuo 已提交
389

390
        """
D
Dong Daxiang 已提交
391
        return get_msg_dict(self.strategy.a_sync_configs)
392

D
Dong Daxiang 已提交
393
    @a_sync_configs.setter
394
    @is_strict_auto
D
Dong Daxiang 已提交
395
    def a_sync_configs(self, configs):
396 397 398
        check_configs_key(
            self.strategy.a_sync_configs, configs, "a_sync_configs"
        )
D
Dong Daxiang 已提交
399
        assign_configs_value(self.strategy.a_sync_configs, configs)
400

401 402 403
    @property
    def trainer_desc_configs(self):
        """
404

405
        Set trainer desc configurations.
406 407 408 409 410 411 412 413

        **Notes**:
            dump_fields_path(str): the path of dump fields

            dump_fields(list(str)): the fields that you want to dump

            dump_param(list(str)): the param that you want to dump

414
            stat_var_names(list(str)):
415 416

        Examples:
417
            .. code-block:: python
418

419 420 421
                import paddle.distributed.fleet as fleet
                role_maker = fleet.PaddleCloudRoleMaker()
                fleet.init(role_maker)
422

423 424 425
                strategy = fleet.DistributedStrategy()
                configs = {"dump_fields_path": "./dump_data", "dump_fields": ["xxx", "yyy"]}
                strategy.trainer_desc_configs = configs
426

427 428
                # code block for defining loss and local optimizer
                # sgd = fleet.distributed_optimizer(optimizer, strategy)
429 430 431 432

        """
        return get_msg_dict(self.strategy.trainer_desc_configs)

433 434 435
    @property
    def adam_d2sum(self):
        """
436

437
        set adam_d2sum
W
wangguanqun 已提交
438
        Default value: False
439 440

        Examples:
441
            .. code-block:: python
442

443 444 445
                import paddle.distributed.fleet as fleet
                role_maker = fleet.PaddleCloudRoleMaker()
                fleet.init(role_maker)
446

447 448
                strategy = fleet.DistributedStrategy()
                strategy.adam_d2sum = True  # by default this is False
449

450 451
                # code block for defining loss and local optimizer
                # sgd = fleet.distributed_optimizer(optimizer, strategy)
452 453 454 455 456 457 458 459 460 461 462

        """
        return self.strategy.adam_d2sum

    @adam_d2sum.setter
    @is_strict_auto
    def adam_d2sum(self, flag):
        if isinstance(flag, bool):
            self.strategy.adam_d2sum = flag
        else:
            raise ValueError(
463 464 465 466
                "The type of `flag` is invalid, expected type is bool, but received {}".format(
                    type(flag)
                )
            )
467

468 469 470
    @trainer_desc_configs.setter
    @is_strict_auto
    def trainer_desc_configs(self, configs):
471 472 473
        check_configs_key(
            self.strategy.trainer_desc_configs, configs, "trainer_desc_configs"
        )
474 475
        assign_configs_value(self.strategy.trainer_desc_configs, configs)

476 477 478
    @property
    def fs_client_param(self):
        """
479

480
        Set fs client configurations.
481 482

        Note:
483
            uri(str): the uri of fs client
484

485
            user(str): the user_name of fs client
486

487
            passwd(str): the passwd of fs client
488

489
            hadoop_bin(str):
490

491
        Examples:
492 493 494 495 496 497 498 499 500 501 502
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                role_maker = fleet.PaddleCloudRoleMaker()
                fleet.init(role_maker)
                strategy = fleet.DistributedStrategy()
                configs = {"uri": "xxx", "user": "xxx", passwd: "xxx"}
                strategy.fs_client_param = configs
                # code block for defining loss and local optimizer
                # sgd = fleet.distributed_optimizer(optimizer, strategy)

503 504 505 506 507 508
        """
        return self.strategy.fs_client_param

    @fs_client_param.setter
    @is_strict_auto
    def fs_client_param(self, configs):
509 510 511
        check_configs_key(
            self.strategy.fs_client_param, configs, "fs_client_param"
        )
512 513 514 515 516 517 518 519 520 521
        assign_configs_value(self.strategy.fs_client_param, configs)

    @property
    def sparse_table_configs(self):
        return self.strategy.downpour_table_param

    @sparse_table_configs.setter
    @is_strict_auto
    def sparse_table_configs(self, configs):
        from google.protobuf.descriptor import FieldDescriptor
522

523 524
        table_param = self.strategy.downpour_table_param

525
        def set_table_config(msg, config_name, configs, index=0):
526 527 528
            for field in msg.DESCRIPTOR.fields:
                name = config_name + "." + field.name
                if field.type == FieldDescriptor.TYPE_MESSAGE:
529
                    logger.debug(f"message: {name}")
530 531 532 533
                    if field.label == FieldDescriptor.LABEL_REPEATED:
                        if name + ".num" not in configs:
                            continue
                        num = configs[name + ".num"]
534
                        logger.debug(f"message num: {name} {num}")
535 536 537 538
                        for i in range(num):
                            data = getattr(msg, field.name).add()
                            set_table_config(data, name, configs, i)
                    else:
539 540 541
                        set_table_config(
                            getattr(msg, field.name), name, configs
                        )
542
                else:
543
                    logger.debug("not message: %s", name)
544 545 546 547 548
                    if name not in configs:
                        continue
                    if field.label == FieldDescriptor.LABEL_REPEATED:
                        getattr(msg, field.name).extend(configs[name])
                    else:
549 550 551 552
                        if type(configs[name]) == list:
                            setattr(msg, field.name, configs[name][index])
                        else:
                            setattr(msg, field.name, configs[name])
553

554
        if not configs:
555
            logger.info("table configs is empty")
556
        else:
557 558 559
            for table_name in configs:
                table_data = table_param.add()
                table_data.table_name = table_name
560 561 562 563 564
                set_table_config(
                    table_data,
                    "table_parameters." + table_name,
                    configs[table_name],
                )
565

566 567
    @sparse_table_configs.setter
    def fleet_desc_configs(self, configs):
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
        support_sparse_key_list = [
            'sparse_table_class',
            'sparse_compress_in_save',
            'sparse_shard_num',
            'sparse_accessor_class',
            'sparse_learning_rate',
            'sparse_initial_g2sum',
            'sparse_initial_range',
            'sparse_weight_bounds',
            'sparse_fea_dim',
            'sparse_embedx_dim',
            'sparse_embedx_threshold',
            'sparse_nonclk_coeff',
            'sparse_click_coeff',
            'sparse_base_threshold',
            'sparse_delta_threshold',
            'sparse_delta_keep_days',
            'sparse_delete_after_unseen_days',
            'sparse_show_click_decay_rate',
            'sparse_delete_threshold',
            'sparse_converter',
            'sparse_deconverter',
            'sparse_enable_cache',
            'sparse_cache_rate',
            'sparse_cache_file_num',
            'sparse_beta1_decay_rate',
            'sparse_beta2_decay_rate',
            'sparse_ada_epsilon',
            'sparse_optimizer',
            'sparse_ssd_unseenday_threshold',
            'embed_sparse_optimizer',
            'embed_sparse_learning_rate',
            'embed_sparse_weight_bounds',
            'embed_sparse_initial_range',
            'embed_sparse_initial_g2sum',
            'embed_sparse_beta1_decay_rate',
            'embed_sparse_beta2_decay_rate',
            'embedx_sparse_optimizer',
            'embedx_sparse_learning_rate',
            'embedx_sparse_weight_bounds',
            'embedx_sparse_initial_range',
            'embedx_sparse_initial_g2sum',
            'embedx_sparse_beta1_decay_rate',
            'embedx_sparse_beta2_decay_rate',
            'feature_learning_rate',
            'nodeid_slot',
L
lxsbupt 已提交
614 615 616 617 618
            'sparse_load_filter_slots',
        ]
        support_sparse_table_class = [
            'DownpourSparseTable',
            'DownpourSparseSSDTable',
619
        ]
620
        support_sparse_accessor_class = [
621 622 623 624 625 626
            'DownpourSparseValueAccessor',
            'DownpourCtrAccessor',
            'DownpourCtrDoubleAccessor',
            'DownpourUnitAccessor',
            'DownpourDoubleUnitAccessor',
            'DownpourCtrDymfAccessor',
627 628 629
        ]
        table_param = self.strategy.downpour_table_param

D
danleifeng 已提交
630
        def add_graph_config(graph, strategy):
631 632 633
            graph.feature_learning_rate = strategy.get(
                'feature_learning_rate', 0.05
            )
D
danleifeng 已提交
634 635
            graph.nodeid_slot = strategy.get('nodeid_slot', 9008)

636
        def sparse_optimizer_config(sgd, strategy, prefix):
637 638 639
            optimizer_name = strategy.get(
                prefix + "sparse_optimizer", "adagrad"
            )
640 641 642 643
            sgd.name = optimizer_name
            if optimizer_name == "naive":
                sgd.name = "SparseNaiveSGDRule"
                sgd.naive.learning_rate = strategy.get(
644 645
                    prefix + 'sparse_learning_rate', 0.05
                )
646
                sgd.naive.initial_range = strategy.get(
647 648 649 650 651
                    prefix + 'sparse_initial_range', 1e-4
                )
                bounds = strategy.get(
                    prefix + 'sparse_weight_bounds', [-10, 10]
                )
652 653 654 655
                sgd.naive.weight_bounds.extend(bounds)
            elif optimizer_name == "adagrad":
                sgd.name = 'SparseAdaGradSGDRule'
                sgd.adagrad.learning_rate = strategy.get(
656 657
                    prefix + 'sparse_learning_rate', 0.05
                )
658
                sgd.adagrad.initial_range = strategy.get(
659 660
                    prefix + 'sparse_initial_range', 1e-4
                )
661 662 663
                if prefix == "embed_":
                    sgd.adagrad.initial_range = 0
                sgd.adagrad.initial_g2sum = strategy.get(
664 665 666 667 668
                    prefix + 'sparse_initial_g2sum', 3
                )
                bounds = strategy.get(
                    prefix + 'sparse_weight_bounds', [-10, 10]
                )
669 670 671 672
                sgd.adagrad.weight_bounds.extend(bounds)
            elif optimizer_name == "std_adagrad":
                sgd.name = 'StdAdaGradSGDRule'
                sgd.adagrad.learning_rate = strategy.get(
673 674
                    prefix + 'sparse_learning_rate', 0.05
                )
675
                sgd.adagrad.initial_range = strategy.get(
676 677
                    prefix + 'sparse_initial_range', 1e-4
                )
678 679 680
                if prefix == "embed_":
                    sgd.adagrad.initial_range = 0
                sgd.adagrad.initial_g2sum = strategy.get(
681 682 683 684 685
                    prefix + 'sparse_initial_g2sum', 3
                )
                bounds = strategy.get(
                    prefix + 'sparse_weight_bounds', [-10, 10]
                )
686 687 688
                sgd.adagrad.weight_bounds.extend(bounds)
            elif optimizer_name == "adam":
                sgd.name = 'SparseAdamSGDRule'
D
danleifeng 已提交
689
                sgd.adam.learning_rate = strategy.get(
690 691
                    prefix + 'sparse_learning_rate', 0.001
                )
D
danleifeng 已提交
692
                sgd.adam.initial_range = strategy.get(
693 694
                    prefix + 'sparse_initial_range', 1e-4
                )
D
danleifeng 已提交
695
                sgd.adam.beta1_decay_rate = strategy.get(
696 697
                    prefix + 'sparse_beta1_decay_rate', 0.9
                )
D
danleifeng 已提交
698
                sgd.adam.beta2_decay_rate = strategy.get(
699 700
                    prefix + 'sparse_beta2_decay_rate', 0.999
                )
D
danleifeng 已提交
701
                sgd.adam.ada_epsilon = strategy.get(
702 703 704 705 706
                    prefix + 'sparse_ada_epsilon', 1e-8
                )
                bounds = strategy.get(
                    prefix + 'sparse_weight_bounds', [-10, 10]
                )
D
danleifeng 已提交
707 708 709
                sgd.adam.weight_bounds.extend(bounds)
            elif optimizer_name == "shared_adam":
                sgd.name = 'SparseSharedAdamSGDRule'
710
                sgd.adam.learning_rate = strategy.get(
711 712
                    prefix + 'sparse_learning_rate', 0.001
                )
713
                sgd.adam.initial_range = strategy.get(
714 715
                    prefix + 'sparse_initial_range', 1e-4
                )
716
                sgd.adam.beta1_decay_rate = strategy.get(
717 718
                    prefix + 'sparse_beta1_decay_rate', 0.9
                )
719
                sgd.adam.beta2_decay_rate = strategy.get(
720 721
                    prefix + 'sparse_beta2_decay_rate', 0.999
                )
722
                sgd.adam.ada_epsilon = strategy.get(
723 724 725 726 727
                    prefix + 'sparse_ada_epsilon', 1e-8
                )
                bounds = strategy.get(
                    prefix + 'sparse_weight_bounds', [-10, 10]
                )
728 729 730 731 732 733
                sgd.adam.weight_bounds.extend(bounds)

        def set_sparse_table_config(table_data, config):
            for key in config:
                if key not in support_sparse_key_list:
                    raise ValueError("strategy key '%s' not support" % (key))
734 735 736
            table_class = config.get(
                "sparse_table_class", "DownpourSparseTable"
            )
737 738
            if table_class not in support_sparse_table_class:
                raise ValueError(
L
lxsbupt 已提交
739
                    "support sparse_table_class: ['DownpourSparseTable, DownpourSparseSSDTable'], but actual %s"
740 741
                    % (table_class)
                )
L
lxsbupt 已提交
742 743 744 745
            if table_class == "DownpourSparseSSDTable":
                table_data.table_class = 'SSDSparseTable'
            else:
                table_data.table_class = 'MemorySparseTable'
746
            table_data.shard_num = config.get('sparse_shard_num', 1000)
747
            table_data.enable_sparse_table_cache = config.get(
748 749
                'sparse_enable_cache', True
            )
750
            table_data.sparse_table_cache_rate = config.get(
751 752
                'sparse_cache_rate', 0.00055
            )
753
            table_data.sparse_table_cache_file_num = config.get(
754 755
                'sparse_cache_file_num', 16
            )
756

757 758 759
            accessor_class = config.get(
                "sparse_accessor_class", "DownpourCtrAccessor"
            )
760 761
            if accessor_class not in support_sparse_accessor_class:
                raise ValueError(
762
                    "support sparse_accessor_class: ['DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor'], but actual %s"
763 764
                    % (accessor_class)
                )
765

766 767
            if accessor_class.find("Double") >= 0:
                table_data.accessor.accessor_class = 'CtrDoubleAccessor'
768 769
            elif accessor_class.find("Dymf") >= 0:
                table_data.accessor.accessor_class = 'CtrDymfAccessor'
770
            else:
771 772 773
                table_data.accessor.accessor_class = 'CtrCommonAccessor'

            if not configs.get("use_cvm", True):
774 775 776 777 778
                table_data.accessor.accessor_class = 'SparseAccessor'

            table_data.accessor.embedx_dim = config.get('sparse_embedx_dim', 8)
            table_data.accessor.fea_dim = table_data.accessor.embedx_dim + 3
            table_data.accessor.embedx_threshold = config.get(
779 780
                'sparse_embedx_threshold', 10
            )
781

782 783 784 785 786
            if accessor_class == 'DownpourUnitAccessor':
                table_data.accessor.ctr_accessor_param.show_scale = False
            else:
                table_data.accessor.ctr_accessor_param.show_scale = True

787
            table_data.accessor.ctr_accessor_param.nonclk_coeff = config.get(
788 789
                'sparse_nonclk_coeff', 0.1
            )
790
            table_data.accessor.ctr_accessor_param.click_coeff = config.get(
791 792
                'sparse_click_coeff', 1
            )
793
            table_data.accessor.ctr_accessor_param.base_threshold = config.get(
794 795
                'sparse_base_threshold', 1.5
            )
796
            table_data.accessor.ctr_accessor_param.delta_threshold = config.get(
797 798
                'sparse_delta_threshold', 0.25
            )
799
            table_data.accessor.ctr_accessor_param.delta_keep_days = config.get(
800 801 802 803 804 805 806 807 808 809 810 811 812 813
                'sparse_delta_keep_days', 16
            )
            table_data.accessor.ctr_accessor_param.show_click_decay_rate = (
                config.get('sparse_show_click_decay_rate', 0.98)
            )
            table_data.accessor.ctr_accessor_param.delete_threshold = (
                config.get('sparse_delete_threshold', 0.8)
            )
            table_data.accessor.ctr_accessor_param.delete_after_unseen_days = (
                config.get('sparse_delete_after_unseen_days', 30)
            )
            table_data.accessor.ctr_accessor_param.ssd_unseenday_threshold = (
                config.get('sparse_ssd_unseenday_threshold', 1)
            )
L
lxsbupt 已提交
814 815 816 817
            load_filter_slots = config.get('sparse_load_filter_slots', [])
            table_data.accessor.ctr_accessor_param.load_filter_slots.extend(
                load_filter_slots
            )
818 819 820 821 822 823 824 825 826 827 828 829 830
            converter = config.get('sparse_converter', "")
            deconverter = config.get('sparse_deconverter', "")

            save_data1 = table_data.accessor.table_accessor_save_param.add()
            save_data1.param = 1
            save_data1.converter = converter
            save_data1.deconverter = deconverter

            save_data2 = table_data.accessor.table_accessor_save_param.add()
            save_data2.param = 2
            save_data2.converter = converter
            save_data2.deconverter = deconverter

831 832 833 834 835 836 837 838 839 840
            if (
                accessor_class == 'DownpourCtrAccessor'
                or accessor_class == 'DownpourCtrDoubleAccessor'
            ):
                sparse_optimizer_config(
                    table_data.accessor.embed_sgd_param, config, ''
                )
                sparse_optimizer_config(
                    table_data.accessor.embedx_sgd_param, config, ''
                )
841
            else:
842 843 844 845 846 847
                sparse_optimizer_config(
                    table_data.accessor.embed_sgd_param, config, 'embed_'
                )
                sparse_optimizer_config(
                    table_data.accessor.embedx_sgd_param, config, 'embedx_'
                )
D
danleifeng 已提交
848
            add_graph_config(table_data.accessor.graph_sgd_param, config)
849 850

        if not configs:
851
            logger.info("fleet desc config is empty")
852 853
        else:
            for table_name in configs:
854 855 856 857
                if (
                    table_name == 'dense_table'
                    or table_name == 'datanorm_table'
                ):
858 859 860 861 862 863 864
                    continue
                if type(configs[table_name]) != dict:
                    continue
                table_data = table_param.add()
                table_data.table_name = table_name
                set_sparse_table_config(table_data, configs[table_name])

865
    @property
866 867 868 869
    def amp(self):
        """
        Indicating whether we are using automatic mixed precision training
        Default Value: False
870

871
        Examples:
1
123malin 已提交
872

873
          .. code-block:: python
874

875
            import paddle.distributed.fleet as fleet
876 877
            strategy = fleet.DistributedStrategy()
            strategy.amp = True # by default this is false
878

879 880
        """
        return self.strategy.amp
881

882
    @amp.setter
883
    @is_strict_auto
884
    def amp(self, flag):
885
        if isinstance(flag, bool):
886
            self.strategy.amp = flag
887
        else:
888
            logger.warning("amp should have value of bool type")
889 890

    @property
891
    def amp_configs(self):
892
        """
893

894 895 896 897
        Set automatic mixed precision training configurations. In general, amp has serveral configurable
        settings that can be configured through a dict.

        **Notes**:
M
mapingshuo 已提交
898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
            init_loss_scaling(float): The initial loss scaling factor. Default 32768.

            use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. Default True.

            incr_every_n_steps(int): Increases loss scaling every n consecutive steps with finite gradients. Default 1000.

            decr_every_n_nan_or_inf(int): Decreases loss scaling every n accumulated steps with nan or inf gradients. Default 2.

            incr_ratio(float): The multiplier to use when increasing the loss scaling. Default 2.0.

            decr_ratio(float): The less-than-one-multiplier to use when decreasing the loss scaling. Default 0.5.

            custom_white_list(list[str]): Users' custom white list which always execution fp16.

            custom_black_list(list[str]): Users' custom black list which forbidden execution fp16.
913

914 915 916 917 918 919 920 921
            custom_black_varnames(list[str]): Users' custom black varibles' names.

            use_pure_fp16(bool): Whether to use the pure fp16 training. Default False.

            use_fp16_guard(bool): Whether to use `fp16_guard` when constructing the program.
                   Default True. Only takes effect when `use_pure_fp16` is turned on.

        Examples 1:
922
            .. code-block:: python
1
123malin 已提交
923

924 925 926 927 928 929
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.amp = True
                strategy.amp_configs = {
                    "init_loss_scaling": 32768,
                    "custom_white_list": ['conv2d']}
930 931

        Examples 2:
932 933 934 935 936 937 938 939 940 941
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.amp = True
                # pure fp16
                strategy.amp_configs = {
                    "init_loss_scaling": 32768,
                    "use_pure_fp16": True
                }
942

943
        """
944
        return get_msg_dict(self.strategy.amp_configs)
945

946
    @amp_configs.setter
947
    @is_strict_auto
948 949 950
    def amp_configs(self, configs):
        check_configs_key(self.strategy.amp_configs, configs, "amp_configs")
        assign_configs_value(self.strategy.amp_configs, configs)
951

952 953 954
    @property
    def asp(self):
        """
955

956 957 958 959
        Indicating whether we are using automatic sparsity training
        Default Value: False

        Examples:
960
            .. code-block:: python
961

962 963 964
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.asp = True # by default this is false
965 966 967 968 969 970 971 972 973 974

        """
        return self.strategy.asp

    @asp.setter
    @is_strict_auto
    def asp(self, flag):
        if isinstance(flag, bool):
            self.strategy.asp = flag
        else:
975
            logger.warning("asp should have value of bool type")
976

977
    @property
978 979 980 981 982 983
    def recompute(self):
        """
        Indicating whether we are using forward recomputation for memory optimization
        Default value: False

        Examples:
984
            .. code-block:: python
1
123malin 已提交
985

986 987 988 989 990
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.recompute = True
                # suppose x and y are names of checkpoint tensors for recomputation
                strategy.recompute_configs = {"checkpoints": ["x", "y"]}
991 992 993

        """
        return self.strategy.recompute
994

995 996
    @property
    def sync_nccl_allreduce(self):
997
        """
998

999 1000 1001 1002
        Indicating whether we are using synchronized all reduce in each communication thread
        We note that system overhead is usually lower when sync_nccl_allreduce = True

        Examples:
1003
            .. code-block:: python
1
123malin 已提交
1004

1005 1006 1007
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.sync_nccl_allreduce = True
1008 1009

        """
1010 1011 1012
        return self.strategy.sync_nccl_allreduce

    @sync_nccl_allreduce.setter
1013
    @is_strict_auto
1014 1015 1016 1017
    def sync_nccl_allreduce(self, flag):
        if isinstance(flag, bool):
            self.strategy.sync_nccl_allreduce = flag
        else:
1018
            logger.warning("sync_nccl_allreduce should have value of bool type")
1019

1020
    @property
1021
    def use_hierarchical_allreduce(self):
1022
        """
1023

1024 1025 1026 1027 1028
        Indicating whether we are using hierarchical allreduce in collective communication
        Hierarchical allreduce often does allreduce within a certain node group and then do
        allreduce among the leaders of each group

        Examples:
1029
            .. code-block:: python
1
123malin 已提交
1030

1031 1032 1033
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.use_hierarchical_allreduce = True
1034 1035

        """
1036
        return self.strategy.use_hierarchical_allreduce
1037

1038
    @use_hierarchical_allreduce.setter
1039
    @is_strict_auto
1040
    def use_hierarchical_allreduce(self, flag):
1041
        if isinstance(flag, bool):
1042
            self.strategy.use_hierarchical_allreduce = flag
1043
        else:
1044 1045
            logger.warning(
                "use_hierarchical_allreduce should have value of bool type"
1046 1047 1048
            )

    @property
1049
    def hierarchical_allreduce_inter_nranks(self):
1050
        """
1051

1052 1053 1054 1055
        Number of ranks for low level node groups in hierarchical allreduce
        Default value: number of GPU cards on each single GPU machine

        Example:
1056
            .. code-block:: python
1
123malin 已提交
1057

1058 1059 1060
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.hierarchical_allreduce_inter_nranks = 8
1061 1062

        """
1063
        return self.strategy.hierarchical_allreduce_inter_nranks
1064

1065
    @hierarchical_allreduce_inter_nranks.setter
1066
    @is_strict_auto
1067 1068 1069
    def hierarchical_allreduce_inter_nranks(self, value):
        if isinstance(value, int):
            self.strategy.hierarchical_allreduce_inter_nranks = value
1070
        else:
1071 1072
            logger.warning(
                "hierarchical_allreduce_inter_nranks should have value of int type"
1073 1074
            )

1075
    @property
1076
    def sync_batch_norm(self):
1077
        """
1078

1079
        Indicating whether we are using sync_batch_norm to do synchronous batch normalization among all training nodes.
1
123malin 已提交
1080

1081 1082 1083
        Default value: False

        Examples:
1084
            .. code-block:: python
1
123malin 已提交
1085

1086 1087 1088
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.sync_batch_norm = True
1089 1090 1091

        """

1092
        return self.strategy.sync_batch_norm
1093

1094
    @sync_batch_norm.setter
1095
    @is_strict_auto
1096
    def sync_batch_norm(self, flag):
1097
        if isinstance(flag, bool):
1098
            self.strategy.sync_batch_norm = flag
1099
        else:
1100
            logger.warning("sync_batch_norm should have value of bool type")
1101 1102 1103

    @property
    def fuse_all_reduce_ops(self):
1104
        """
1105

1106 1107 1108 1109
        Indicating whether we are using fuse_all_reduce_ops for gradient fusion during backward phase of training
        Default value: True

        Examples:
1110
            .. code-block:: python
1
123malin 已提交
1111

1112 1113 1114
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.fuse_all_reduce_ops = False
1115 1116

        """
1117 1118 1119
        return self.strategy.fuse_all_reduce_ops

    @fuse_all_reduce_ops.setter
1120
    @is_strict_auto
1121 1122 1123 1124
    def fuse_all_reduce_ops(self, flag):
        if isinstance(flag, bool):
            self.strategy.fuse_all_reduce_ops = flag
        else:
1125
            logger.warning("fuse_all_reduce_ops should have value of bool type")
1126

1127 1128
    @property
    def fuse_grad_size_in_MB(self):
1129
        """
1130

1131 1132 1133 1134 1135
        Specifying the size of gradient to fuse in Mega-Bytes

        Default value: 32

        Examples:
1136
            .. code-block:: python
1
123malin 已提交
1137

1138 1139 1140
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.fuse_grad_size_in_MB = 50
1
123malin 已提交
1141

1142
        """
1143 1144 1145
        return self.strategy.fuse_grad_size_in_MB

    @fuse_grad_size_in_MB.setter
1146
    @is_strict_auto
1147 1148 1149 1150
    def fuse_grad_size_in_MB(self, value):
        if isinstance(value, int):
            self.strategy.fuse_grad_size_in_MB = value
        else:
1151
            logger.warning("fuse_grad_size_in_MB should have value of int type")
1152

1153 1154 1155
    @property
    def last_comm_group_size_MB(self):
        """
1156

1157 1158 1159
        Specifying the size of gradient to fuse in Mega-Bytes when
        the last group of each batch communicates. Making the last group
        small is useful to improve performance.
1160 1161 1162 1163

        Default value: 1

        Examples:
1164 1165 1166 1167 1168
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.last_comm_group_size_MB = 2
1169

1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
        """
        return self.strategy.last_comm_group_size_MB

    @last_comm_group_size_MB.setter
    @is_strict_auto
    def last_comm_group_size_MB(self, value):
        if value > 0:
            self.strategy.last_comm_group_size_MB = value
        else:
            raise ValueError("last_comm_group_size_MB should be greater than 0")

1181 1182 1183
    @property
    def find_unused_parameters(self):
        """
1184

1185
        Indicating whether we are using find_unused_parameters to
1186 1187
        find unused parameters in DataParallel.

1188
        Default value: False
1189 1190

        Examples:
1191
            .. code-block:: python
1192

1193 1194 1195
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.find_unused_parameters = True
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206

        """

        return self.strategy.find_unused_parameters

    @find_unused_parameters.setter
    @is_strict_auto
    def find_unused_parameters(self, flag):
        if isinstance(flag, bool):
            self.strategy.find_unused_parameters = flag
        else:
1207 1208
            logger.warning(
                "find_unused_parameters should have value of bool type"
1209
            )
1210

1211 1212 1213 1214 1215
    @property
    def _fuse_grad_size_in_TFLOPS(self):
        return self.strategy.fuse_grad_size_in_TFLOPS

    @_fuse_grad_size_in_TFLOPS.setter
1216
    @is_strict_auto
1217 1218 1219 1220
    def _fuse_grad_size_in_TFLOPS(self, value):
        if isinstance(value, float):
            self.strategy.fuse_grad_size_in_TFLOPS = value
        else:
1221 1222
            logger.warning(
                "fuse_grad_size_in_TFLOPS should have value of float type"
1223 1224
            )

1225
    @property
1226
    def nccl_comm_num(self):
1227
        """
1228

1229 1230 1231 1232 1233
        Specifying the number of NCCL communicator

        Default value: 1

        Examples:
1234
            .. code-block:: python
1
123malin 已提交
1235

1236 1237 1238
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.nccl_comm_num = 2
1
123malin 已提交
1239

1240 1241
        """

1242
        return self.strategy.nccl_comm_num
1243

1244
    @nccl_comm_num.setter
1245
    @is_strict_auto
1246
    def nccl_comm_num(self, value):
1247
        if isinstance(value, int):
1248
            self.strategy.nccl_comm_num = value
1249
        else:
1250
            logger.warning("nccl_comm_num should have value of int type")
1251

1252
    @recompute.setter
1253
    @is_strict_auto
1254
    def recompute(self, flag):
1255
        if isinstance(flag, bool):
1256
            self.strategy.recompute = flag
1257
        else:
1258
            logger.warning("recompute should have value of bool type")
1259 1260

    @property
1261 1262
    def recompute_configs(self):
        """
1263

1264 1265
        Set recompute configurations.

J
JZ-LIANG 已提交
1266 1267 1268 1269
        **Note**:
        checkpoints(list): list of string name of checkpoints. In general, the recompute
        strategy of current implementation should have some manually assign checkpoints.

1270
        enable_offload(bool): enable recompute checkpoints offload feature. this feature
J
JZ-LIANG 已提交
1271 1272 1273 1274 1275 1276
        will offload the checkpoint to host memory to allow even larger batch size. since
        the memcpy from host to device takes time, it is a trade off between larger batch
        size and training speed.

        checkpoint_shape(list): list of int that specific the shape of checkpoint. so far
        recompute-offload requires that all checkpoint to be same shape, and every dimension
1277
        specific here should be determined ("-1" is not allowed).
1278

1279
        Examples:
1280
            .. code-block:: python
1
123malin 已提交
1281

1282 1283 1284 1285 1286 1287 1288
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.recompute = True
                strategy.recompute_configs = {
                    "checkpoints": ["x", "y"],
                    "enable_offload": True,
                    "checkpoint_shape": [100, 512, 1024] }
1289 1290 1291 1292 1293

        """
        return get_msg_dict(self.strategy.recompute_configs)

    @recompute_configs.setter
1294
    @is_strict_auto
1295
    def recompute_configs(self, configs):
1296 1297 1298
        check_configs_key(
            self.strategy.recompute_configs, configs, "checkpoint_configs"
        )
1299
        assign_configs_value(self.strategy.recompute_configs, configs)
1300

1301 1302 1303
    @property
    def sharding(self):
        """
1304

1305
        Indicating whether we are using sharding Optimizer for memory
1306
        optimization. We implement the sharding optimizer following the ZeRO-DP
J
JZ-LIANG 已提交
1307 1308
        idea from [ZeRO: Memory Optimizations Toward Training Trillion Parameter Models](https://arxiv.org/abs/1910.02054).
        Model parameters and Optimizer State are sharded into different ranks allowing to fit larger model.
1309

1310 1311
        In Hybrid parallelism scenario, we use sharding config as uniform API to set each parallelism.

1312 1313 1314
        Default value: False

        Examples:
1315
            .. code-block:: python
1
123malin 已提交
1316

1317 1318 1319
                import paddle.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.sharding = True
1
123malin 已提交
1320

1321 1322 1323 1324 1325 1326 1327 1328 1329
        """
        return self.strategy.sharding

    @sharding.setter
    @is_strict_auto
    def sharding(self, flag):
        if isinstance(flag, bool):
            self.strategy.sharding = flag
        else:
1330
            logger.warning("sharding should have value of bool type")
1331 1332 1333 1334

    @property
    def sharding_configs(self):
        """
1335

1336
        Set sharding configurations.
1337 1338

        **Note**:
1339 1340
            sharding_segment_strategy(string, optional): strategy used to segment the program(forward & backward operations). two strategise are
            available: "segment_broadcast_MB" and "segment_anchors". segment is a concept used in sharding to overlap computation and
1341 1342
            communication. Default is segment_broadcast_MB.

1343
            segment_broadcast_MB(float, optional): segment by the parameters broadcast volume. sharding will introduce parameter broadcast operations into program, and
1344 1345 1346 1347
            after every segment_broadcast_MB size parameter being broadcasted, the program will be cutted into one segment.
            This configuration will affect the communication speed in sharding training, and should be an empirical value decided by your model size and network topology.
            Only enable when sharding_segment_strategy = segment_broadcast_MB. Default is 32.0 .

1348
            segment_anchors(list): list of anchors used to segment the program, which allows a finner control of program segmentation.
1349 1350 1351 1352 1353 1354
            this strategy is experimental by now. Only enable when sharding_segment_strategy = segment_anchors.

            sharding_degree(int, optional): specific the number of gpus within each sharding parallelism group; and sharding will be turn off if sharding_degree=1.  Default is 8.

            gradient_merge_acc_step(int, optional): specific the accumulation steps in gradient merge; and gradient merge will be turn off if gradient_merge_acc_step=1.  Default is 1.

1355
            optimize_offload(bool, optional): enable the optimizer offload which will offload the moment vars to Host memory in order to saving GPU memory for fitting larger model.
1356 1357 1358 1359 1360
            the moment var will be prefetch from and offloaded to Host memory during update stage. it is a stragtegy that trades off between training speed and GPU memory, and is recommened to be turn on only when gradient_merge_acc_step large, where
            the number of time of update stage will be relatively small compared with forward&backward's.  Default is False.

            dp_degree(int, optional): specific the number of data parallelism group; when dp_degree >= 2, it will introduce dp_degree ways data parallelism as the outer parallelsim for the inner parallelsim. User is responsible to ensure global_world_size = mp_degree * sharding_degree * pp_degree * dp_degree. Default is 1.

1361
            mp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each megatron parallelism group; and megatron parallelism will turn be off if mp_degree=1.  Default is 1.
1362

1363
            pp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each pipeline parallelism group; and pipeline parallelism will turn be off if pp_degree=1.  Default is 1.
1364

1365
            pp_allreduce_in_optimize(bool, optional): [Hybrid parallelism ONLY] move the allreduce operations from backward stage to update(optimize) stage when pipeline parallelsim is on.
1366
            This configuration will affect the communication speed of Hybrid parallelism training depeneded on network topology. this strategy is experimental by now..  Default is False.
J
JZ-LIANG 已提交
1367

1368 1369 1370
            optimize_cast(bool, optional): [Hybrid parallelism ONLY] Move the cast op of AMP which cast fp32 param to fp16 param to optimizer. optimize_cast will persist fp16 param, it
            will take more memory, but will be faster, trade space for time. Recommend to turn on only when using pipeline or gradient_merge_acc_step large.

J
JZ-LIANG 已提交
1371

1372
        Examples:
1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
            .. code-block:: python

                # sharding-DP, 2 nodes with 8 gpus per node
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.sharding = True
                strategy.sharding_configs = {
                    "sharding_segment_strategy": "segment_broadcast_MB",
                    "segment_broadcast_MB": 32,
                    "sharding_degree": 8,
                    "dp_degree": 2,
                    "gradient_merge_acc_step": 4,
                    }
1
123malin 已提交
1386

1387 1388 1389 1390 1391 1392
        """
        return get_msg_dict(self.strategy.sharding_configs)

    @sharding_configs.setter
    @is_strict_auto
    def sharding_configs(self, configs):
1393 1394 1395
        check_configs_key(
            self.strategy.sharding_configs, configs, "sharding_configs"
        )
1396 1397
        assign_configs_value(self.strategy.sharding_configs, configs)

1398 1399 1400
    @property
    def without_graph_optimization(self):
        """
1401

1402 1403 1404
        Run program using Executor other than ParallelExecutor.

        Examples:
1405
            .. code-block:: python
1406

1407 1408 1409
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.without_graph_optimization = True
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419

        """
        return self.strategy.without_graph_optimization

    @without_graph_optimization.setter
    @is_strict_auto
    def without_graph_optimization(self, flag):
        if isinstance(flag, bool):
            self.strategy.without_graph_optimization = flag
        else:
1420 1421
            logger.warning(
                "without_graph_optimization should have value of bool type"
1422 1423
            )

1424 1425 1426
    @property
    def _calc_comm_same_stream(self):
        """
1427

1428 1429 1430
        This based on raw_program_optimizer program
        Set whether use same stream for calc and comm when fuse allreduce
        The default value for the calc_comm_same_stream is False
1431

1432
        Examples:
1433 1434 1435 1436 1437 1438
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.calc_comm_same_stream = True

1439 1440 1441 1442 1443 1444 1445 1446 1447
        """
        return self.strategy.calc_comm_same_stream

    @_calc_comm_same_stream.setter
    @is_strict_auto
    def _calc_comm_same_stream(self, same):
        if isinstance(same, bool):
            self.strategy.calc_comm_same_stream = same
        else:
1448 1449
            logger.warning(
                "calc_comm_same_stream should have value of boolean type"
1450 1451
            )

1452 1453 1454
    @property
    def fuse_grad_merge(self):
        """
1455

1456 1457 1458
        Set whether fuse the grad for gradient merge.
        Note: this flag will only effect the gradient merge under pipeline mode
        The default value for the fuse_grad_merge is False
1459

1460
        Examples:
1461 1462 1463 1464 1465 1466
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.fuse_param_grad = True

1467 1468 1469 1470 1471 1472 1473 1474 1475
        """
        return self.strategy.fuse_grad_merge

    @fuse_grad_merge.setter
    @is_strict_auto
    def fuse_grad_merge(self, fuse_grad_merge):
        if isinstance(fuse_grad_merge, bool):
            self.strategy.fuse_grad_merge = fuse_grad_merge
        else:
1476
            logger.warning("fuse_grad_merge should have value of boolean type")
1477

1478 1479 1480
    @property
    def fuse_grad_size_in_num(self):
        """
1481

1482
        This based on raw_program_optimizer program and allreduce the num of the fused op
1483

1484
        Examples:
1485 1486 1487 1488 1489 1490 1491
            .. code-block:: python

                import paddle.distributed.fleet as fleet

                strategy = fleet.DistributedStrategy()
                strategy.fuse_grad_size_in_num = 2

1492 1493 1494 1495 1496 1497 1498 1499 1500
        """
        return self.strategy.fuse_grad_size_in_num

    @fuse_grad_size_in_num.setter
    @is_strict_auto
    def fuse_grad_size_in_num(self, num):
        if isinstance(num, int):
            self.strategy.fuse_grad_size_in_num = num
        else:
1501 1502
            logger.warning(
                "fuse_grad_size_in_num should have value of int32 type"
1503
            )
1504

1505
    @property
1506 1507
    def pipeline(self):
        """
1508

1509 1510 1511 1512 1513 1514
        Indicating whether we are using pipeline parallelism for distributed training.
        Current implementation mainly focus on single GPU machine pipeline parallelism and
        data parallelism across GPU machine. The pipeline information is indicated through
        device_guard information in user-defined program.

        Examples:
1515
            .. code-block:: python
1
123malin 已提交
1516

1517 1518 1519
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.pipeline = True
1520 1521 1522

        """
        return self.strategy.pipeline
1523

1524 1525 1526 1527 1528 1529 1530 1531 1532 1533
    @property
    def is_fl_ps_mode(self):
        return self.strategy.is_fl_ps_mode

    @is_fl_ps_mode.setter
    @is_strict_auto
    def is_fl_ps_mode(self, flag):
        if isinstance(flag, bool):
            self.strategy.is_fl_ps_mode = flag
        else:
1534
            logger.warning("is_fl_ps_mode should have value of bool type")
1535

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545
    @property
    def is_with_coordinator(self):
        return self.strategy.with_coordinator

    @is_with_coordinator.setter
    @is_strict_auto
    def is_with_coordinator(self, flag):
        if isinstance(flag, bool):
            self.strategy.with_coordinator = flag
        else:
1546
            logger.warning("with_coordinator should have value of bool type")
1547

1548
    @pipeline.setter
1549
    @is_strict_auto
1550
    def pipeline(self, flag):
1551
        if isinstance(flag, bool):
1552
            self.strategy.pipeline = flag
1553
        else:
1554
            logger.warning("pipeline should have value of bool type")
1555 1556

    @property
1557 1558
    def pipeline_configs(self):
        """
1559

1560 1561
        Set pipeline parallelism configurations. In pipeline parallelism,
        different parts of neural networks are running on different GPUS.
1562
        There are Tensor queue buffer between each pair of neighborhood GPUS
1563 1564 1565
        that are responsible for synchronizing hidden Tensor results between
        GPUs. Pipeline parallelism consists of serveral producer-consumer style
        hardware pairs, such as GPU-GPU, CPU-GPU, GPU-XPU. The best way to speedup
1566
        pipeline parallelism is to make the size of Tensor in Tensor queue smaller,
1567
        so that we will have a faster producer for downstream consumers.
1568

1569 1570
        **Notes**:
            **Detailed arguments for pipeline_configs**
M
mapingshuo 已提交
1571

1572
            **micro_batch_size**: the number of small batches in each user defined batch
1573

1574
        Examples:
1575
            .. code-block:: python
1
123malin 已提交
1576

1577 1578 1579 1580
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.pipeline = True
                strategy.pipeline_configs = {"micro_batch_size": 12}
1581

1582
        """
1583

1584
        return get_msg_dict(self.strategy.pipeline_configs)
1585

1586
    @pipeline_configs.setter
1587
    @is_strict_auto
1588
    def pipeline_configs(self, configs):
1589 1590 1591
        check_configs_key(
            self.strategy.pipeline_configs, configs, "pipeline_configs"
        )
1592
        assign_configs_value(self.strategy.pipeline_configs, configs)
1593

L
lilong12 已提交
1594 1595 1596
    @property
    def tensor_parallel(self):
        """
1597

L
lilong12 已提交
1598 1599 1600
        Indicating whether we are using tensor parallel for distributed training.

        Examples:
1601
            .. code-block:: python
L
lilong12 已提交
1602

1603 1604 1605
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.tensor_parallel = True
L
lilong12 已提交
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615

        """
        return self.strategy.tensor_parallel

    @tensor_parallel.setter
    @is_strict_auto
    def tensor_parallel(self, flag):
        if isinstance(flag, bool):
            self.strategy.tensor_parallel = flag
        else:
1616
            logger.warning("tensor_parallel should have value of bool type")
L
lilong12 已提交
1617 1618 1619 1620

    @property
    def tensor_parallel_configs(self):
        """
1621

L
lilong12 已提交
1622 1623 1624 1625
        Set tensor_parallel configurations.

        **Notes**:
            **Detailed arguments for tensor_parallel_configs**
1626

L
lilong12 已提交
1627
            **tensor_parallel_degree**: degree of tensor parallel
1628

1629 1630
            **tensor_init_seed**: parameter initialization random seed

L
lilong12 已提交
1631 1632

        Examples:
1633
            .. code-block:: python
L
lilong12 已提交
1634

1635 1636 1637 1638 1639
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.tensor_parallel = True
                strategy.tensor_parallel_configs = {"tensor_parallel_degree": 4,
                                                    "tensor_init_seed": 123}
L
lilong12 已提交
1640 1641 1642 1643 1644 1645 1646

        """
        return get_msg_dict(self.strategy.tensor_parallel_configs)

    @tensor_parallel_configs.setter
    @is_strict_auto
    def tensor_parallel_configs(self, configs):
1647 1648 1649 1650 1651
        check_configs_key(
            self.strategy.tensor_parallel_configs,
            configs,
            "tensor_parallel_configs",
        )
L
lilong12 已提交
1652 1653
        assign_configs_value(self.strategy.tensor_parallel_configs, configs)

1654 1655 1656
    @property
    def hybrid_configs(self):
        """
1657

1658
        Dynamic graph hybrid parallel strategy configuration. Three-way hybrid parallelism
1659 1660 1661 1662 1663
        needs to meet the following relationships

        total_number_GPUs = dp_degree * mp_degree * pp_degree

        **Note**:
1664
            **dp_degree(int)**: set number of GPUs in a data parallel group. Default -1.
1665
                                    This value should be an integer greater than 0.
1666
                                    If it is not set, or set to -1, its value will be inferred
1667 1668
                                    based on the total number of cards.

1669 1670 1671
            **mp_degree(int)**: set number of GPUs in a model parallel group. Default 1

            **pp_degree(int)**: set number of GPUs in a pipeline parallel group. Default 1
1672

1673 1674
            **order(list(string))**: set hybrid parallel dimensions, the order is from outside to inside. Default ['dp','pp','sharding','mp']

1675
        Examples:
1676 1677 1678 1679 1680 1681 1682
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.hybrid_configs = {
                    "dp_degree": 1,
                    "mp_degree": 2,
1683 1684
                    "pp_degree": 1,
                    "order":['dp','pp','sharding','mp']}
1685

1686 1687 1688 1689 1690
        """
        return get_msg_dict(self.strategy.hybrid_configs)

    @hybrid_configs.setter
    def hybrid_configs(self, configs):
1691 1692 1693 1694 1695
        hybrid_config = copy.deepcopy(configs)
        if "order" in hybrid_config:
            self.hybrid_parallel_order = hybrid_config["order"]
            hybrid_config.pop('order')

1696
        check_configs_key(
1697
            self.strategy.hybrid_configs, hybrid_config, "hybrid_configs"
1698
        )
1699 1700 1701 1702 1703 1704

        if "mp_configs" in configs:
            assign_configs_value(
                self.strategy.hybrid_configs.mp_configs, configs["mp_configs"]
            )
            configs.pop("mp_configs")
1705 1706 1707 1708 1709 1710
        if "pp_configs" in configs:
            assign_configs_value(
                self.strategy.hybrid_configs.pp_configs, configs["pp_configs"]
            )
            configs.pop("pp_configs")

1711 1712
        assign_configs_value(self.strategy.hybrid_configs, configs)

1713
    @property
1714
    def localsgd(self):
1715
        """
1716

M
mapingshuo 已提交
1717 1718 1719
        Indicating whether we are using Local SGD training. Default Value: False
        For more details, please refer to
        `Don't Use Large Mini-Batches, Use Local SGD <https://arxiv.org/pdf/1808.07217.pdf>`_.
1720 1721 1722


        Examples:
1723
            .. code-block:: python
1
123malin 已提交
1724

1725 1726 1727
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.localsgd = True # by default this is false
1728 1729

        """
1730
        return self.strategy.localsgd
1731

1732
    @localsgd.setter
1733
    @is_strict_auto
1734 1735 1736
    def localsgd(self, flag):
        if isinstance(flag, bool):
            self.strategy.localsgd = flag
1737
        else:
1738
            logger.warning("localsgd should have value of bool type")
1739 1740

    @property
1741
    def localsgd_configs(self):
1742
        """
1743

1744 1745 1746 1747
        Set LocalSGD training configurations. LocalSGD has a configurable
        setting that can be configured through a dict.

        **Notes**:
M
mapingshuo 已提交
1748
            k_steps(int) The local steps for training before parameter synchronization. Default 1.
1749
            begin_step(int) The step of beginning training by localsgd. Default 1.
1750 1751

        Examples:
1752
            .. code-block:: python
1
123malin 已提交
1753

1754 1755 1756 1757 1758
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.localsgd = True
                strategy.localsgd_configs = {"k_steps": 4,
                                            "begin_step": 30}
1759 1760 1761

        """

1762
        return get_msg_dict(self.strategy.localsgd_configs)
1763

1764
    @localsgd_configs.setter
1765
    @is_strict_auto
1766
    def localsgd_configs(self, configs):
1767 1768 1769
        check_configs_key(
            self.strategy.localsgd_configs, configs, "localsgd_configs"
        )
1770
        assign_configs_value(self.strategy.localsgd_configs, configs)
1771

1772 1773 1774
    @property
    def adaptive_localsgd(self):
        """
1775

1776
        Indicating whether we are using Adaptive Local SGD training. Default Value: False
1777
        For more details, please refer to `Adaptive Communication Strategies to Achieve
1778 1779 1780
        the Best Error-Runtime Trade-off in Local-Update SGD <https://arxiv.org/pdf/1810.08313.pdf>`_.

        Examples:
1781
            .. code-block:: python
1
123malin 已提交
1782

1783 1784 1785
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.adaptive_localsgd = True # by default this is false
1786 1787

        """
1788
        return self.strategy.adaptive_localsgd
1789 1790 1791 1792 1793

    @adaptive_localsgd.setter
    @is_strict_auto
    def adaptive_localsgd(self, flag):
        if isinstance(flag, bool):
1794
            self.strategy.adaptive_localsgd = flag
1795
        else:
1796
            logger.warning("adaptive_localsgd should have value of bool type")
1797 1798 1799 1800

    @property
    def adaptive_localsgd_configs(self):
        """
1801

1802 1803 1804 1805 1806 1807 1808
        Set AdaptiveLocalSGD training configurations. AdaptiveLocalSGD has a configurable
        setting that can be configured through a dict.

        **Notes**:
            init_k_steps(int) The initial steps for training before adaptive localsgd.
                              Then, the adaptive localsgd method will modify init_k_steps automatically.
                              Default 1.
1809

1810
            begin_step(int) The step of beginning training by adaptive localsgd. Default 1.
1811 1812

        Examples:
1813
            .. code-block:: python
1
123malin 已提交
1814

1815 1816 1817 1818 1819
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.adaptive_localsgd = True
                strategy.adaptive_localsgd_configs = {"init_k_steps": 1,
                                                    "begin_step": 30}
1820 1821 1822 1823 1824 1825 1826 1827

        """

        return get_msg_dict(self.strategy.adaptive_localsgd_configs)

    @adaptive_localsgd_configs.setter
    @is_strict_auto
    def adaptive_localsgd_configs(self, configs):
1828 1829 1830 1831 1832
        check_configs_key(
            self.strategy.adaptive_localsgd_configs,
            configs,
            "adaptive_localsgd_configs",
        )
1833 1834
        assign_configs_value(self.strategy.adaptive_localsgd_configs, configs)

1835
    @property
1836
    def dgc(self):
1837
        """
1838

1839 1840 1841 1842 1843 1844
        Indicating whether we are using Deep Gradient Compression training. For more details, please refer to
        [Deep Gradient Compression](https://arxiv.org/abs/1712.01887).

        Default Value: False

        Examples:
1845
            .. code-block:: python
1
123malin 已提交
1846

1847 1848 1849
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.dgc = True # by default this is false
1850 1851

        """
1852
        return self.strategy.dgc
1853

1854
    @dgc.setter
1855
    @is_strict_auto
1856 1857 1858
    def dgc(self, flag):
        if isinstance(flag, bool):
            self.strategy.dgc = flag
1859
        else:
1860
            logger.warning("dgc should have value of bool type")
1861 1862

    @property
1863
    def dgc_configs(self):
1864
        r"""
1865

1866 1867 1868 1869
        Set Deep Gradient Compression training configurations. In general, dgc has serveral configurable
        settings that can be configured through a dict.

        **Notes**:
M
mapingshuo 已提交
1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
            rampup_begin_step(int): The beginning step from which gradient compression is implemented. Default 0.

            rampup_step(int): Time steps used in sparsity warm-up periods. Default is 1. \
                    For example, if the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 100, \
                    it will use 0.75 at 0~19 steps, and 0.9375 at 20~39 steps, and so on. And when reach sparsity array \
                    ends, it will use 0.999 then and after.

            sparsity(list[float]): Get top important element from gradient tensor, the ratio is (1 - sparsity). \
                    Default is [0.999]. For example, if the sparsity is [0.99, 0.999], the top [1%, 0.1%] important \
                    element will be transmitted.
1880 1881

        Examples:
1882
            .. code-block:: python
1
123malin 已提交
1883

1884 1885 1886 1887
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.dgc = True
                strategy.dgc_configs = {"rampup_begin_step": 1252}
1888 1889

        """
1890
        return get_msg_dict(self.strategy.dgc_configs)
1891

1892
    @dgc_configs.setter
1893
    @is_strict_auto
1894 1895 1896
    def dgc_configs(self, configs):
        check_configs_key(self.strategy.dgc_configs, configs, "dgc_configs")
        assign_configs_value(self.strategy.dgc_configs, configs)
1897

1898 1899 1900
    @property
    def fp16_allreduce(self):
        """
1901

1902 1903 1904 1905
        Indicating whether we are using fp16 gradient allreduce training
        Default Value: False

        Examples:
1906
            .. code-block:: python
1
123malin 已提交
1907

1908
                import paddle.distributed.fleet as fleet
1909

1910 1911
                strategy = fleet.DistributedStrategy()
                strategy.fp16_allreduce = True # by default this is false
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922

        """
        return self.strategy.fp16_allreduce

    @fp16_allreduce.setter
    @is_strict_auto
    def fp16_allreduce(self, flag):
        if not isinstance(flag, bool):
            raise TypeError('fp16_allreduce must be value of bool type')
        self.strategy.fp16_allreduce = flag

1923
    @property
1924
    def gradient_merge(self):
1925
        """
1926

1927 1928 1929 1930 1931 1932 1933 1934 1935 1936
        Gradient Merge, also called as Gradient Accumulation,
        is a strategy for large batch training. With this strategy,
        model parameter will not be updated until user-defined steps.
        For each step, the forward network and the backward network
        will run to calculate the gradient of model parameters.
        For every k step, the optimization network will run,
        applying a specific optimization method (such as SGD, Adam)
        to model parameters.

        Examples:
1937
            .. code-block:: python
1
123malin 已提交
1938

1939 1940 1941 1942
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.gradient_merge = True
                strategy.gradient_merge_configs = {"k_steps": 4, "avg": True}
M
mapingshuo 已提交
1943

1944
        """
1945
        return self.strategy.gradient_merge
1946

1947
    @gradient_merge.setter
1948
    @is_strict_auto
1949
    def gradient_merge(self, flag):
1950
        if isinstance(flag, bool):
1951
            self.strategy.gradient_merge = flag
1952
        else:
1953
            logger.warning("gradient_merge should have value of bool type")
1954 1955 1956

    @property
    def gradient_merge_configs(self):
1957
        """
1958

1959
        the key-value configs of distribute_strategy
M
mapingshuo 已提交
1960 1961 1962 1963 1964 1965 1966

        **Note**:
            k_steps(int): the update period of the parameters.

            avg(bool): whether to average the gradients of each mini-batch, the default value is `True`

        Examples:
1967
            .. code-block:: python
1
123malin 已提交
1968

1969 1970 1971 1972
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.gradient_merge = True
                strategy.gradient_merge_configs = {"k_steps": 4, "avg": True}
M
mapingshuo 已提交
1973

1974
        """
1975 1976 1977
        return get_msg_dict(self.strategy.gradient_merge_configs)

    @gradient_merge_configs.setter
1978
    @is_strict_auto
1979
    def gradient_merge_configs(self, configs):
1980 1981 1982
        check_configs_key(
            self.strategy.gradient_merge_configs, configs, "gradient_configs"
        )
1983
        assign_configs_value(self.strategy.gradient_merge_configs, configs)
1984 1985

    @property
1986
    def lars(self):
1987
        """
1988

1989 1990
        Set lars configurations. lars is used to deal with the convergence problems when the global
        batch size is larger than 8k.  For more details, please refer to
1991 1992 1993 1994 1995
        [Large Batch Training of Convolutional Networks](https://arxiv.org/abs/1708.03888).

        Default Value: False

        Examples:
1996
            .. code-block:: python
1
123malin 已提交
1997

1998 1999 2000
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.lars = True # by default this is false
2001 2002

        """
2003
        return self.strategy.lars
2004

2005
    @lars.setter
2006
    @is_strict_auto
2007
    def lars(self, flag):
2008
        if isinstance(flag, bool):
2009
            self.strategy.lars = flag
2010
        else:
2011
            logger.warning("lars should have value of bool type")
2012

2013 2014
    @property
    def lars_configs(self):
2015
        """
2016

2017 2018 2019 2020 2021
        Set Lars training configurations.

        **Notes**:
        **lars_coeff (float)**: trust ratio in lars formula.
        **lars_weight_decay** (float): weight decay coefficient in lars formula.
2022 2023
        **epsilon (float)**: argument is used to avoid potential devision-by-zero
        when compute the local lr;
2024 2025 2026 2027
        **exclude_from_weight_decay ([string])**: is a list of name strings of layers which
        will be exclude from weight decay in lars formula.

        Examples:
2028
            .. code-block:: python
1
123malin 已提交
2029

2030 2031 2032 2033 2034 2035 2036 2037 2038
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.lars = True
                strategy.lars_configs = {
                            "lars_coeff": 0.01,
                            "lars_weight_decay": 0.0005,
                            "epsilon": 0,
                            "exclude_from_weight_decay": ['batch_norm', '.b_0']
                        }
M
mapingshuo 已提交
2039

2040
        """
2041 2042 2043
        return get_msg_dict(self.strategy.lars_configs)

    @lars_configs.setter
2044
    @is_strict_auto
2045 2046 2047 2048
    def lars_configs(self, configs):
        check_configs_key(self.strategy.lars_configs, configs, "lars_configs")
        assign_configs_value(self.strategy.lars_configs, configs)

2049
    @property
2050
    def lamb(self):
2051
        """
2052

2053 2054 2055
        Set lamb configurations. lamb is used to deal with the convergence problems for large
        batch size training, specially for attention-related model like BERT. For more details,
        please refer to
2056 2057 2058
        [Large Batch Optimization for Deep Learning: Training BERT in 76 minutes](https://arxiv.org/abs/1904.00962).

        Default Value: False
1
123malin 已提交
2059

2060
        Examples:
2061
            .. code-block:: python
1
123malin 已提交
2062

2063 2064 2065
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.lamb = True # by default this is false
2066 2067 2068

        """

2069
        return self.strategy.lamb
2070

2071
    @lamb.setter
2072
    @is_strict_auto
2073
    def lamb(self, flag):
2074
        if isinstance(flag, bool):
2075
            self.strategy.lamb = flag
2076
        else:
2077
            logger.warning("lamb should have value of bool type")
2078

2079 2080
    @property
    def lamb_configs(self):
2081
        """
2082

2083 2084 2085 2086 2087 2088 2089 2090
        Set Lars training configurations.

        **Notes**:
        **lamb_weight_decay** (float): weight decay coefficient in lamb formula.
        **exclude_from_weight_decay ([string])**: is a list of name strings of layers which
        will be exclude from weight decay in lamb formula.

        Examples:
2091 2092 2093 2094 2095 2096 2097 2098 2099
            .. code-block:: python

                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.lamb = True
                strategy.lamb_configs = {
                        'lamb_weight_decay': 0.01,
                        'exclude_from_weight_decay': [],
                    }
1
123malin 已提交
2100

2101
        """
2102 2103 2104
        return get_msg_dict(self.strategy.lamb_configs)

    @lamb_configs.setter
2105
    @is_strict_auto
2106 2107 2108 2109
    def lamb_configs(self, configs):
        check_configs_key(self.strategy.lamb_configs, configs, "lamb_configs")
        assign_configs_value(self.strategy.lamb_configs, configs)

2110 2111
    @property
    def elastic(self):
2112
        """
2113

2114 2115
        Indicating whether we want to do current distributed training on clusters with elastic resources.
        Currently, this is configuration is not valid.
2116

2117
        """
2118 2119 2120
        return self.strategy.elastic

    @elastic.setter
2121
    @is_strict_auto
2122 2123 2124 2125
    def elastic(self, flag):
        if isinstance(flag, bool):
            self.strategy.elastic = flag
        else:
2126
            logger.warning("elastic should have value of bool type")
2127 2128 2129

    @property
    def auto(self):
2130
        """
2131

2132
        Indicating whether we are using auto-parallel configuration
2133
        This feature is currently an experimental feature. Currently,
2134 2135 2136 2137 2138 2139
        auto-parallelism can be used only when a user does not set any other
        strategy configs except auto. For details, please reference the following
        code example
        Default Value: False

        Examples:
2140
            .. code-block:: python
1
123malin 已提交
2141

2142 2143 2144
                import paddle
                paddle.enable_static()
                import paddle.distributed.fleet as fleet
2145

2146 2147 2148 2149
                strategy = fleet.DistributedStrategy()
                strategy.auto = True
                # if set other strategy at the same time, auto will not apply
                # strategy.amp = True
2150

2151 2152
                optimizer = paddle.optimizer.SGD(learning_rate=0.01)
                optimizer = fleet.distributed_optimizer(optimizer, strategy)
2153 2154

        """
2155 2156 2157 2158 2159 2160 2161
        return self.strategy.auto

    @auto.setter
    def auto(self, flag):
        if isinstance(flag, bool):
            self.strategy.auto = flag
        else:
2162
            logger.warning("auto should have value of bool type")
2163

2164 2165 2166
    @property
    def semi_auto(self):
        """
2167

2168
        Indicating whether we are using semi-auto parallel function
2169
        This feature is currently an experimental feature. Currently,
2170 2171 2172 2173 2174 2175
        auto-parallelism can be used only when a user does not set any other
        strategy configs except semi-auto. For details, please reference the following
        code example
        Default Value: False

        Examples:
2176
            .. code-block:: python
2177

2178 2179 2180
                import paddle
                paddle.enable_static()
                import paddle.distributed.fleet as fleet
2181

2182 2183 2184 2185
                strategy = fleet.DistributedStrategy()
                strategy.semi_auto = True
                # if set other strategy at the same time, auto will not apply
                # strategy.amp = True
2186

2187 2188
                optimizer = paddle.optimizer.SGD(learning_rate=0.01)
                optimizer = fleet.distributed_optimizer(optimizer, strategy)
2189 2190 2191 2192 2193 2194 2195 2196 2197

        """
        return self.strategy.semi_auto

    @semi_auto.setter
    def semi_auto(self, flag):
        if isinstance(flag, bool):
            self.strategy.semi_auto = flag
        else:
2198
            logger.warning("semi-auto should have value of bool type")
2199

Z
zhaoyingli 已提交
2200 2201 2202
    @property
    def auto_search(self):
        """
2203

Z
zhaoyingli 已提交
2204 2205 2206
        Indicating whether we are using auto-search parallel function
        For details, please reference the following code example
        Default Value: False
2207

Z
zhaoyingli 已提交
2208
        Examples:
2209 2210 2211 2212 2213 2214 2215 2216 2217
            .. code-block:: python

                import paddle

                paddle.enable_static()
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.auto_search = True

Z
zhaoyingli 已提交
2218 2219 2220 2221 2222 2223 2224 2225
        """
        return self.strategy.auto_search

    @auto_search.setter
    def auto_search(self, flag):
        if isinstance(flag, bool):
            self.strategy.auto_search = flag
        else:
2226
            logger.warning("auto-search should have value of bool type")
Z
zhaoyingli 已提交
2227

2228 2229 2230
    @property
    def split_data(self):
        """
2231

2232 2233
        Indicating whether we split the data. If True, we split the data.
        Default Value: True
2234

2235
        Examples:
2236 2237 2238 2239 2240 2241 2242 2243 2244
            .. code-block:: python

                import paddle

                paddle.enable_static()
                import paddle.distributed.fleet as fleet
                strategy = fleet.DistributedStrategy()
                strategy.split_data = True

2245 2246 2247 2248 2249 2250 2251 2252
        """
        return self.strategy.split_data

    @split_data.setter
    def split_data(self, flag):
        if isinstance(flag, bool):
            self.strategy.split_data = flag
        else:
2253
            logger.warning("split_data should have value of bool type")
2254

2255 2256 2257
    @property
    def qat(self):
        """
2258

2259 2260
        Indicating whether we are using quantization training
        Default Value: False
2261

2262 2263 2264 2265 2266 2267 2268 2269
        """
        return self.strategy.qat

    @qat.setter
    def qat(self, flag):
        if isinstance(flag, bool):
            self.strategy.qat = flag
        else:
2270
            logger.warning("qat should have value of bool type")
2271 2272 2273 2274

    @property
    def qat_configs(self):
        """
2275

2276 2277 2278 2279 2280 2281 2282 2283 2284 2285
        Set quantization training configurations. In general, qat has serveral configurable
        settings that can be configured through a dict.

        **Notes**:
            channel_wise_abs_max(bool): Whether to use `per_channel` quantization training. Default is True.

            weight_bits(int): quantization bit number for weight. Default is 8.

            activation_bits(int): quantization bit number for activation. Default is 8.

2286
            not_quant_pattern(list[str]): When the skip pattern is detected in an op's name scope,
2287 2288 2289 2290 2291
                the corresponding op will not be quantized.

            algo(str): Other quantization training algorithm.

        Exampless:
2292
            .. code-block:: python
2293

2294
                import paddle.distributed.fleet as fleet
2295

2296 2297 2298 2299 2300 2301 2302
                strategy = fleet.DistributedStrategy()
                strategy.qat = True
                strategy.qat_configs = {
                    "channel_wise_abs_max": True,
                    "weight_bits": 8,
                    "activation_bits: 8,
                    "not_quant_pattern": ['skip_quant']}
2303 2304 2305 2306 2307 2308 2309 2310 2311

        """
        return get_msg_dict(self.strategy.qat_configs)

    @qat_configs.setter
    def qat_configs(self, configs):
        check_configs_key(self.strategy.qat_configs, configs, "qat_configs")
        assign_configs_value(self.strategy.qat_configs, configs)

K
kuizhiqing 已提交
2312 2313 2314
    @property
    def heter_ccl_mode(self):
        """
2315

K
kuizhiqing 已提交
2316 2317 2318 2319 2320 2321
        Indicating whether we are using heter_ccl_mode for model training.
        This feature is currently an experimental feature. Currently,
        heter_ccl_mode can be used only for dataparallel with dygraph mode.
        Default Value: False

        Examples:
2322
            .. code-block:: python
K
kuizhiqing 已提交
2323

2324 2325
                import paddle
                import paddle.distributed.fleet as fleet
K
kuizhiqing 已提交
2326

2327 2328
                strategy = fleet.DistributedStrategy()
                strategy.heter_ccl_mode = True
K
kuizhiqing 已提交
2329

2330 2331 2332
                # for initialize parallel env, only need to call
                paddle.distributed.init_parallel_env()
                # then the heterogenous context will be created.
K
kuizhiqing 已提交
2333 2334 2335 2336 2337 2338 2339 2340 2341

        """
        return self.strategy.heter_ccl_mode

    @heter_ccl_mode.setter
    def heter_ccl_mode(self, flag):
        if isinstance(flag, bool):
            self.strategy.heter_ccl_mode = flag
        else:
2342
            logger.warning("heter_ccl_mode should have value of bool type")
K
kuizhiqing 已提交
2343

2344 2345
    @property
    def cudnn_exhaustive_search(self):
2346
        """
2347

2348 2349 2350 2351 2352 2353 2354
        Indicating whether to use exhaustive search method to choose convolution algorithms.
        Exhaustive search attempts all cuDNN algorithms to choose the fastest algorithm.
        This method is time-consuming, the choosed algorithm will be cached for the given layer specifications.
        Once the layer specifications (like batch size, feature map size) are changed, it will search again.
        Default Value: True

        Examples:
2355
            .. code-block:: python
1
123malin 已提交
2356

2357 2358 2359
                import paddle
                paddle.enable_static()
                import paddle.distributed.fleet as fleet
2360

2361 2362 2363 2364 2365
                strategy = fleet.DistributedStrategy()
                strategy.cudnn_exhaustive_search = False

                optimizer = paddle.optimizer.SGD(learning_rate=0.01)
                optimizer = fleet.distributed_optimizer(optimizer, strategy)
2366 2367

        """
2368 2369 2370
        return self.strategy.cudnn_exhaustive_search

    @cudnn_exhaustive_search.setter
2371
    @is_strict_auto
2372 2373 2374 2375
    def cudnn_exhaustive_search(self, flag):
        if isinstance(flag, bool):
            self.strategy.cudnn_exhaustive_search = flag
        else:
2376 2377
            logger.warning(
                "cudnn_exhaustive_search should have value of bool type"
2378 2379 2380 2381
            )

    @property
    def conv_workspace_size_limit(self):
2382
        """
2383

2384 2385 2386 2387 2388 2389 2390
        The workspace limit size in MB unit for choosing cuDNN convolution algorithms.
        The inner funciton of cuDNN obtain the fastest suited algorithm that fits within this memory limit.
        Usually, large workspace size may lead to choose faster algorithms,
        but significant increasing memory workspace. Users need to trade-off between memory and speed.
        Default Value: 4000

        Examples:
2391
            .. code-block:: python
1
123malin 已提交
2392

2393 2394 2395
                import paddle
                paddle.enable_static()
                import paddle.distributed.fleet as fleet
2396

2397 2398
                strategy = fleet.DistributedStrategy()
                strategy.conv_workspace_size_limit = 1024
2399

2400 2401
                optimizer = paddle.optimizer.SGD(learning_rate=0.01)
                optimizer = fleet.distributed_optimizer(optimizer, strategy)
1
123malin 已提交
2402

2403
        """
2404 2405 2406
        return self.strategy.conv_workspace_size_limit

    @conv_workspace_size_limit.setter
2407
    @is_strict_auto
2408 2409 2410 2411
    def conv_workspace_size_limit(self, value):
        if isinstance(value, int):
            self.strategy.conv_workspace_size_limit = value
        else:
2412 2413
            logger.warning(
                "conv_workspace_size_limit should have value of int type"
2414 2415 2416 2417
            )

    @property
    def cudnn_batchnorm_spatial_persistent(self):
2418
        """
2419

2420 2421 2422 2423 2424
        Indicates whether to use the mode CUDNN_BATCHNORM_SPATIAL_PERSISTENT function in batchnorm.
        This is only useful in cudnn.
        Default Value: True

        Examples:
2425
            .. code-block:: python
1
123malin 已提交
2426

2427 2428 2429
                import paddle
                paddle.enable_static()
                import paddle.distributed.fleet as fleet
2430

2431 2432
                strategy = fleet.DistributedStrategy()
                strategy.cudnn_batchnorm_spatial_persistent = True
2433

2434 2435
                optimizer = paddle.optimizer.SGD(learning_rate=0.01)
                optimizer = fleet.distributed_optimizer(optimizer, strategy)
2436 2437

        """
2438 2439 2440
        return self.strategy.cudnn_batchnorm_spatial_persistent

    @cudnn_batchnorm_spatial_persistent.setter
2441
    @is_strict_auto
2442 2443 2444 2445
    def cudnn_batchnorm_spatial_persistent(self, flag):
        if isinstance(flag, bool):
            self.strategy.cudnn_batchnorm_spatial_persistent = flag
        else:
2446 2447
            logger.warning(
                "cudnn_batchnorm_spatial_persistent should have value of bool type"
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
            )

    def _enable_env(self):
        strategy = self.strategy
        keys = [
            "FLAGS_cudnn_batchnorm_spatial_persistent",
            "FLAGS_conv_workspace_size_limit",
            "FLAGS_cudnn_exhaustive_search",
            "FLAGS_sync_nccl_allreduce",
            "FLAGS_fuse_parameter_memory_size",
            "FLAGS_fuse_parameter_groups_size",
        ]
        values = [
            bool(strategy.cudnn_batchnorm_spatial_persistent),
            int(strategy.conv_workspace_size_limit),
            bool(strategy.cudnn_exhaustive_search),
            bool(strategy.sync_nccl_allreduce),
            int(strategy.fuse_grad_size_in_MB),
            int(strategy.fuse_grad_size_in_TFLOPS),
        ]

        for i, key in enumerate(keys):
2470 2471
            if _global_flags().is_public(key):
                _global_flags()[key] = values[i]
2472

2473 2474 2475 2476 2477 2478
    def _is_strict_auto(self):
        global non_auto_func_called
        if self.strategy.auto and non_auto_func_called:
            return True
        return False

2479
    def __repr__(self):
2480 2481 2482 2483 2484 2485
        spacing = 2
        max_k = 38
        max_v = 38

        length = max_k + max_v + spacing

2486
        h1_format = "    " + f"|{{:^{length}s}}|\n"
2487
        h2_format = "    " + "|{{:>{}s}}{}{{:^{}s}}|\n".format(
2488 2489
            max_k, " " * spacing, max_v
        )
2490 2491 2492 2493 2494 2495 2496 2497 2498

        border = "    +" + "".join(["="] * length) + "+"
        line = "    +" + "".join(["-"] * length) + "+"

        draws = border + "\n"
        draws += h1_format.format("")
        draws += h1_format.format("DistributedStrategy Overview")
        draws += h1_format.format("")

D
Dong Daxiang 已提交
2499
        fields = self.strategy.DESCRIPTOR.fields
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513
        str_res = ""

        env_draws = line + "\n"
        for f in fields:
            if "build_strategy" in f.name or "execution_strategy" in f.name:
                continue
            if "_configs" in f.name:
                continue
            else:
                if isinstance(getattr(self.strategy, f.name), bool):
                    if hasattr(self.strategy, f.name + "_configs"):
                        if getattr(self.strategy, f.name):
                            draws += border + "\n"
                            draws += h1_format.format(
2514
                                f"{f.name}=True <-> {f.name}_configs"
2515
                            )
2516
                            draws += line + "\n"
2517 2518 2519
                            my_configs = getattr(
                                self.strategy, f.name + "_configs"
                            )
2520
                            config_fields = my_configs.DESCRIPTOR.fields
R
risemeup1 已提交
2521 2522 2523 2524 2525 2526 2527 2528 2529
                            protobuf_version = google.protobuf.__version__
                            if protobuf_version >= "4.21.0":
                                RepeatedScalarContainer = (
                                    google._upb._message.RepeatedScalarContainer
                                )
                            else:
                                RepeatedScalarContainer = (
                                    google.protobuf.pyext._message.RepeatedScalarContainer
                                )
2530 2531
                            for ff in config_fields:
                                if isinstance(
2532
                                    getattr(my_configs, ff.name),
R
risemeup1 已提交
2533
                                    RepeatedScalarContainer,
2534
                                ):
2535 2536 2537
                                    values = getattr(my_configs, ff.name)
                                    for i, v in enumerate(values):
                                        if i == 0:
2538
                                            draws += h2_format.format(
2539 2540
                                                ff.name, str(v)
                                            )
2541
                                        else:
2542
                                            draws += h2_format.format(
2543 2544
                                                "", str(v)
                                            )
2545 2546 2547
                                else:
                                    draws += h2_format.format(
                                        ff.name,
2548 2549
                                        str(getattr(my_configs, ff.name)),
                                    )
2550 2551
                    else:
                        env_draws += h2_format.format(
2552 2553
                            f.name, str(getattr(self.strategy, f.name))
                        )
2554 2555
                else:
                    env_draws += h2_format.format(
2556 2557 2558 2559 2560 2561 2562 2563 2564
                        f.name, str(getattr(self.strategy, f.name))
                    )

        result_res = (
            draws
            + border
            + "\n"
            + h1_format.format("Environment Flags, Communication Flags")
        )
2565 2566 2567 2568 2569 2570 2571
        result_res += env_draws

        build_strategy_str = border + "\n"
        build_strategy_str += h1_format.format("Build Strategy")
        build_strategy_str += line + "\n"

        fields = self.strategy.build_strategy.DESCRIPTOR.fields
D
Dong Daxiang 已提交
2572
        for f in fields:
2573
            build_strategy_str += h2_format.format(
2574 2575
                f.name, str(getattr(self.strategy.build_strategy, f.name))
            )
2576 2577 2578 2579 2580 2581 2582 2583
        build_strategy_str += border + "\n"

        execution_strategy_str = h1_format.format("Execution Strategy")
        execution_strategy_str += line + "\n"

        fields = self.strategy.execution_strategy.DESCRIPTOR.fields
        for f in fields:
            execution_strategy_str += h2_format.format(
2584 2585
                f.name, str(getattr(self.strategy.execution_strategy, f.name))
            )
2586 2587 2588 2589
        execution_strategy_str += border + "\n"

        result_res += build_strategy_str + execution_strategy_str
        return result_res