raw_program_optimizer.py

#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and

from __future__ import print_function
from __future__ import division
import os
import collections
import numpy as np

import paddle.fluid as fluid
from paddle.fluid import core, unique_name
from paddle.fluid.dygraph import Layer, LayerList
from ..base.private_helper_function import wait_server_ready
from .meta_optimizer_base import MetaOptimizerBase
from .common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY, CollectiveHelper, is_loss_grad_op, is_backward_op, is_optimizer_op


class RawProgramOptimizer(MetaOptimizerBase):
    def __init__(self, optimizer):
        super(RawProgramOptimizer, self).__init__(optimizer)
        self.inner_opt = optimizer
        self.meta_optimizers_white_list = [
            "RecomputeOptimizer",
            "AMPOptimizer",
        ]
        self.meta_optimizers_black_list = ["GraphExecutionOptimizer", ]
        self.global_ring_id = 0

    def _set_basic_info(self, loss, role_maker, user_defined_optimizer,
                        user_defined_strategy):
        super(RawProgramOptimizer, self)._set_basic_info(
            loss, role_maker, user_defined_optimizer, user_defined_strategy)
        self.without_graph_optimization = user_defined_strategy.without_graph_optimization
        self.fuse_all_reduce_ops = user_defined_strategy.fuse_all_reduce_ops
        if self.fuse_all_reduce_ops:
            self.fuse_grad_size_in_num = user_defined_strategy.fuse_grad_size_in_num

    def _can_apply(self):
        if not self.role_maker._is_collective:
            return False

        if self.without_graph_optimization == True:
            return True
        return False

    def _disable_strategy(self, dist_strategy):
        dist_strategy.without_graph_optimization = False

    def _enable_strategy(self, dist_strategy, context):
        dist_strategy.without_graph_optimization = True

    def _broadcast_params(self, ring_id):
        block = self.startup_program.global_block()
        param = None
        for param in block.iter_parameters():
            if param.is_distributed:
                continue

            block.append_op(
                type='c_broadcast',
                inputs={'X': param},
                outputs={'Out': param},
                attrs={
                    'ring_id': ring_id,
                    'root': 0,
                    OP_ROLE_KEY: OpRole.Forward
                })

        if not param: return  # no parameter on this device
        block.append_op(
            type='c_sync_comm_stream',
            inputs={'X': param},
            outputs={'Out': param},
            attrs={'ring_id': ring_id,
                   OP_ROLE_KEY: OpRole.Forward})

    def _get_process_group_info(self):
        # global ring info
        self.global_endpoints = self.endpoints
        self.global_rank = self.rank
        self.global_nranks = self.nranks

    def _init_process_group(self):
        self._get_process_group_info()
        collective_helper = CollectiveHelper(self.role_maker, wait_port=False)
        # Create global ring for all gpus (ring_id = 0)
        collective_helper._init_communicator(
            self.startup_program, self.current_endpoint, self.global_endpoints,
            self.global_rank, self.global_ring_id, True, self.global_ring_id,
            True)
        self._broadcast_params(self.global_ring_id)

    def minimize_impl(self,
                      loss,
                      startup_program=None,
                      parameter_list=None,
                      no_grad_set=None):
        self.endpoints = self.role_maker._get_trainer_endpoints()
        self.current_endpoint = self.endpoints[self.role_maker._worker_index()]
        self.rank = self.role_maker._worker_index()
        self.nranks = self.role_maker._worker_num()
        if startup_program is None:
            startup_program = fluid.default_startup_program()
        self.startup_program = startup_program

        block = loss.block
        program = block.program
        self.main_program = program

        optimize_ops, params_grads = self.inner_opt.minimize(
            loss, startup_program, parameter_list, no_grad_set)
        if self.nranks == 1:
            return optimize_ops, params_grads
        self._init_process_group()

        self.main_program = program
        if self.nranks > 1:
            self._transpile_main_program(loss)
        return optimize_ops, params_grads

    def _transpile_main_program(self, loss):
        self._insert_loss_grad_ops(loss)
        if self.fuse_all_reduce_ops and core.is_compiled_with_npu():
            self._calc_stream = True
            self._allreduce_fusion_program()
        else:
            self._insert_allreduce_ops()

    def _insert_loss_grad_ops(self, loss):
        """
        In order to keep the learning rate consistent in different numbers of
        training workers, we scale the loss grad by the number of workers
        """
        block = self.main_program.global_block()
        for idx, op in reversed(list(enumerate(block.ops))):
            if is_loss_grad_op(op):
                loss_grad_var = block.vars[op.output_arg_names[0]]
                block._insert_op(
                    idx + 1,
                    type='scale',
                    inputs={'X': loss_grad_var},
                    outputs={'Out': loss_grad_var},
                    attrs={
                        'scale': 1.0 / self.nranks,
                        OP_ROLE_KEY: OpRole.Backward
                    })

    def _insert_allreduce_ops(self):
        block = self.main_program.global_block()
        ring_id = self.global_ring_id
        grad = None
        for idx, op in reversed(list(enumerate(block.ops))):
            if is_backward_op(op) and \
                    OP_ROLE_VAR_KEY in op.attr_names:
                op_role_var = op.attr(OP_ROLE_VAR_KEY)
                if len(op_role_var) == 0:
                    continue
                assert len(op_role_var) % 2 == 0
                offset = 1
                for i in range(0, len(op_role_var), 2):
                    param_name = op_role_var[i]
                    param = block.var(param_name)
                    grad_name = op_role_var[i + 1]
                    grad = block.var(grad_name)
                    if param.is_distributed:
                        continue

                    block._insert_op(
                        idx + offset,
                        type='c_sync_calc_stream',
                        inputs={'X': grad},
                        outputs={'Out': grad},
                        attrs={OP_ROLE_KEY: OpRole.Backward, })
                    offset += 1
                    block._insert_op(
                        idx + offset,
                        type='c_allreduce_sum',
                        inputs={'X': grad},
                        outputs={'Out': grad},
                        attrs={
                            'ring_id': ring_id,
                            OP_ROLE_KEY: OpRole.Backward
                        })

        if grad is None:
            return

        for idx, op in enumerate(block.ops):
            if is_optimizer_op(op):
                block._insert_op(
                    idx,
                    type='c_sync_comm_stream',
                    inputs={'X': grad},
                    outputs={'Out': grad},
                    attrs={'ring_id': ring_id,
                           OP_ROLE_KEY: OpRole.Backward})
                break

    # TODO(Liu yuang): ADD CUDA allreduce_fusion fuction.
    # This function helps reduce the input of allreduce by integrating can save communication time.
    def _allreduce_fusion_program(self):
        block = self.main_program.global_block()
        ring_id = self.global_ring_id
        record_idx, allreduce_input_vars, allreduce_output_vars = [], [], []
        block_ops = len(list(enumerate(block.ops)))

        for idx, op in reversed(list(enumerate(block.ops))):
            if is_backward_op(op) and \
                    OP_ROLE_VAR_KEY in op.attr_names:
                op_role_var = op.attr(OP_ROLE_VAR_KEY)
                if len(op_role_var) == 0:
                    continue
                assert len(op_role_var) % 2 == 0
                for i in range(0, len(op_role_var), 2):
                    param_name = op_role_var[i]
                    param = block.var(param_name)
                    grad_name = op_role_var[i + 1]
                    grad = block.var(grad_name)
                    if param.is_distributed:
                        continue
                    if ".cast_fp16@GRAD" in grad_name:
                        param_name = param_name + ".cast_fp16"
                        if not block.has_var(param_name):
                            raise ValueError("op cast name error {}".format(
                                op.type))
                        else:
                            param = block.var(param_name)

                    if len(allreduce_output_vars) == 0:
                        allreduce_output_vars.append([grad])
                        allreduce_input_vars.append([param])
                        if self.fuse_grad_size_in_num == 1:
                            record_idx.append([idx, idx])
                            continue
                        record_idx.append([-2, idx])
                    elif len(allreduce_output_vars[
                            -1]) == self.fuse_grad_size_in_num:
                        allreduce_output_vars.append([grad])
                        allreduce_input_vars.append([param])
                        if self.fuse_grad_size_in_num == 1:
                            record_idx.append([idx, idx])
                            continue
                        if idx != block_ops - 1:
                            record_idx.append([-2, idx])
                    else:
                        allreduce_output_vars[-1].append(grad)
                        allreduce_input_vars[-1].append(param)
                        record_idx[-1][0] = idx

                if record_idx[-1][0] == -2:
                    record_idx[-1][0] = record_idx[-1][1]

        assert len(allreduce_output_vars) == len(
            record_idx
        ), "It has different lens between the allreduce_output_vars and record_idx."

        if not allreduce_output_vars or not allreduce_input_vars:
            return

        self.vars = collections.OrderedDict()
        index, offset_pos, pos, offset = 0, 0, 0, 0
        start, end = record_idx[index]
        men_list = [end, start]

        # Here we need to explain the flag. When integrating OP, we will encounter different groups of the same Op.
        # Because we insert coalesce tensor in reverse ops,
        # we need to use flag to record whether the current OP has been inserted into coalesce tensor。
        # For example:
        # [(3, 2), (2, 2), (1, 0)], (3, 2), (2, 2) using same op, but in different groups.

        for idx, op in reversed(list(enumerate(block.ops))):
            if idx == start:
                pos = 0
                flag = True if end == men_list[-1] else False
                offset = offset_pos if flag else 0
                done_output_vars, done_input_vars = self._split_fuction(
                    allreduce_output_vars[index], allreduce_input_vars[index])
                for id_, done_output_var in enumerate(done_output_vars):
                    if flag:
                        tmp_var = block.create_var(
                            name=unique_name.generate(
                                'FusedOutput_{}_{}'.format(start, id_ +
                                                           offset)),
                            dtype=done_output_var[0].dtype,
                            persistable=False,
                            stop_gradient=True)
                        self.vars['FusedOutput_{}_{}'.format(start, id_ +
                                                             offset)] = tmp_var

                        block._insert_op(
                            idx + id_ + offset,
                            type="coalesce_tensor",
                            inputs={"Input": done_input_vars[id_]},
                            outputs={
                                "Output": done_output_var,
                                "FusedOutput": tmp_var
                            },
                            attrs={
                                "copy_data": False,
                                "use_align": True,
                                "dtype": done_output_var[0].dtype
                            })
                        pos += 1
                    else:
                        tmp_var = block.create_var(
                            name=unique_name.generate(
                                'FusedOutput_{}_{}'.format(start, id_)),
                            dtype=done_output_var[0].dtype,
                            persistable=False,
                            stop_gradient=True)
                        self.vars['FusedOutput_{}_{}'.format(start,
                                                             id_)] = tmp_var

                        block._insert_op(
                            idx + id_,
                            type="coalesce_tensor",
                            inputs={"Input": done_input_vars[id_]},
                            outputs={
                                "Output": done_output_var,
                                "FusedOutput": tmp_var
                            },
                            attrs={
                                "copy_data": False,
                                "use_align": True,
                                "dtype": done_output_var[0].dtype
                            })
                        pos += 1
                offset_pos = pos

                # TODO(Liu yuang): ADD CUDA and NPU's EVENT and c_allreduce_sum.
                for id_ in range(len(done_output_vars)):
                    if flag:
                        block._insert_op(
                            end + id_ + pos + 1,
                            type='c_allreduce_sum',
                            inputs={
                                'X': self.vars['FusedOutput_{}_{}'.format(
                                    start, id_ + offset)]
                            },
                            outputs={
                                'Out': self.vars['FusedOutput_{}_{}'.format(
                                    start, id_ + offset)]
                            },
                            attrs={
                                'ring_id': ring_id,
                                'use_calc_stream': True
                                if self._calc_stream else False,
                                OP_ROLE_KEY: OpRole.Backward
                            })
                    else:
                        block._insert_op(
                            end + id_ + pos + 1,
                            type='c_allreduce_sum',
                            inputs={
                                'X': self.vars['FusedOutput_{}_{}'.format(start,
                                                                          id_)]
                            },
                            outputs={
                                'Out': self.vars['FusedOutput_{}_{}'.format(
                                    start, id_)]
                            },
                            attrs={
                                'ring_id': ring_id,
                                'use_calc_stream': True
                                if self._calc_stream else False,
                                OP_ROLE_KEY: OpRole.Backward
                            })
                index += 1
                men_list.append(end)
                men_list.append(start)
                if len(record_idx) == index:
                    start = end = -1
                    continue
                start, end = record_idx[index]

        if not self._calc_stream:
            for idx, op in enumerate(block.ops):
                if is_optimizer_op(op):
                    block._insert_op(
                        idx,
                        type='c_sync_comm_stream',
                        inputs={'X': block.create_var()},
                        outputs={'Out': block.create_var()},
                        attrs={
                            'ring_id': ring_id,
                            OP_ROLE_KEY: OpRole.Backward
                        })
                    break

    # Integrate grads of the same type to form a combination. If skip_comb is selected, will return grads of the same group.
    # For example:[(fp16, fp16), (fp32), (fp16)] -> [(fp16, fp16, fp16), (fp32)]
    def _split_fuction(self,
                       allreduce_output_vars,
                       allreduce_input_vars,
                       skip_comb=True):
        input_vars, final_input_vars, output_vars, final_output_vars = [], [], [], []
        if len(allreduce_output_vars) - 1 == 0:
            final_output_vars.append(allreduce_output_vars)
            final_input_vars.append(allreduce_input_vars)
            return final_output_vars, final_input_vars

        for idx in range(len(allreduce_input_vars) - 1):
            if allreduce_input_vars[idx].dtype == allreduce_input_vars[idx +
                                                                       1].dtype:
                input_vars.append(allreduce_input_vars[idx])
                if idx == len(allreduce_input_vars) - 2:
                    input_vars.append(allreduce_input_vars[idx + 1])
                    final_input_vars.append(input_vars)
            else:
                input_vars.append(allreduce_input_vars[idx])
                final_input_vars.append(input_vars)
                input_vars = []
                if idx == len(allreduce_input_vars) - 2:
                    input_vars.append(allreduce_input_vars[idx + 1])
                    final_input_vars.append(input_vars)

        for idx in range(len(allreduce_output_vars) - 1):
            if allreduce_output_vars[idx].dtype == allreduce_output_vars[
                    idx + 1].dtype:
                output_vars.append(allreduce_output_vars[idx])
                if idx == len(allreduce_output_vars) - 2:
                    output_vars.append(allreduce_output_vars[idx + 1])
                    final_output_vars.append(output_vars)
            else:
                output_vars.append(allreduce_output_vars[idx])
                final_output_vars.append(output_vars)
                output_vars = []
                if idx == len(allreduce_output_vars) - 2:
                    output_vars.append(allreduce_output_vars[idx + 1])
                    final_output_vars.append(output_vars)
        if skip_comb:
            input_fp16_vars, input_fp32_vars, output_fp16_vars, output_fp32_vars = [], [], [], []
            for final_input_var in final_input_vars:
                if final_input_var[0].dtype == core.VarDesc.VarType.FP16:
                    input_fp16_vars.extend(final_input_var)
                else:
                    input_fp32_vars.extend(final_input_var)

            for final_output_var in final_output_vars:
                if final_output_var[0].dtype == core.VarDesc.VarType.FP16:
                    output_fp16_vars.extend(final_output_var)
                else:
                    output_fp32_vars.extend(final_output_var)
            final_output_vars, final_input_vars = [], []
            if output_fp16_vars:
                final_output_vars.append(output_fp16_vars)
            if output_fp32_vars:
                final_output_vars.append(output_fp32_vars)
            if input_fp16_vars:
                final_input_vars.append(input_fp16_vars)
            if input_fp32_vars:
                final_input_vars.append(input_fp32_vars)

        return final_output_vars, final_input_vars