ps_trainer_pass.py 61.3 KB
Newer Older
Z
ziyoujiyi 已提交
1
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2
#
Z
ziyoujiyi 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
6
#
Z
ziyoujiyi 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
Z
ziyoujiyi 已提交
9 10 11 12 13 14 15 16
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import paddle
17
from ..ps.utils.public import *  # noqa: F403
Z
ziyoujiyi 已提交
18
from paddle.framework import core
19
from paddle.distributed.passes.pass_base import PassBase, register_pass
Z
ziyoujiyi 已提交
20 21
from paddle.fluid.transpiler.details.program_utils import delete_ops
from paddle.fluid.transpiler.collective import SingleProcessMultiThread
22
from _collections import defaultdict
23
from paddle.fluid.framework import Program, Parameter
Z
ziyoujiyi 已提交
24 25 26 27 28


@register_pass("append_send_ops_pass")
class AppendSendOpsPass(PassBase):  # 该 pass 被多种模式复用
    def __init__(self):
29
        super().__init__()
Z
ziyoujiyi 已提交
30 31 32 33 34 35 36

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

37 38 39
    def _append_send_op(
        self, program, union_vars, queue, is_sparse, table_id, ps_mode
    ):
Z
ziyoujiyi 已提交
40 41 42 43 44 45 46 47 48 49 50
        if queue == STEP_COUNTER:
            send_input_vars = []
        else:
            send_input_vars = [
                program.global_block().vars[union_var]
                for union_var in union_vars
            ]

        dummy_output = []
        if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]:
            dummy_output = program.global_block().create_var(
51 52 53 54 55 56 57 58 59 60 61 62 63
                name=framework.generate_control_dev_var_name()
            )
        program.global_block().append_op(
            type="send",
            inputs={"X": send_input_vars},
            outputs={"Out": dummy_output},
            attrs={
                "send_varnames": [queue],
                "is_sparse": is_sparse,
                "table_id": table_id,
                RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
            },
        )
Z
ziyoujiyi 已提交
64 65 66

        return dummy_output

67
    def _append_barrier_op(self, program, dummys, trainer_id):
68 69 70 71 72 73 74 75 76 77
        program.global_block().append_op(
            type="send_barrier",
            inputs={"X": dummys},
            outputs={"Out": []},
            attrs={
                "trainer_id": trainer_id,
                "half_async": True,
                RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
            },
        )
Z
ziyoujiyi 已提交
78 79 80 81

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        ps_mode = attrs['ps_mode']
82
        # if ps_mode == DistributedMode.GEO:
83 84
        #   send_ctx = get_geo_trainer_send_context(attrs)  # geo 模式, 没必要
        send_ctx = get_the_one_send_context(
85 86
            attrs, split_dense_table=attrs['is_heter_ps_mode']
        )  # async、sync 等各种模式
87

Z
ziyoujiyi 已提交
88
        dummys = []
89
        for merged_name, send in send_ctx.items():  # embedding_0.w_0@GRAD
Z
ziyoujiyi 已提交
90 91
            if send.is_sparse() and ps_mode != DistributedMode.GEO:
                continue
92 93
            if (not send.is_sparse()) and ps_mode == DistributedMode.GEO:
                continue
94 95
            if send.program_id() != id(attrs['loss'].block.program):
                continue
96 97
            if len(send.remote_sparse_ids()) > 0:
                continue
Z
ziyoujiyi 已提交
98 99 100
            is_sparse = 1 if send.is_sparse() else 0
            is_sparse = 2 if send.is_distributed() else is_sparse
            dummys.append(
101 102 103 104 105 106 107 108 109
                self._append_send_op(
                    main_program,
                    send.origin_varnames(),
                    merged_name,
                    is_sparse,
                    send.table_id(),
                    ps_mode,
                )
            )
Z
ziyoujiyi 已提交
110
        if ps_mode in [DistributedMode.SYNC, DistributedMode.HALF_ASYNC]:
111 112
            trainer_id = get_role_id(attrs['role_maker'])
            self._append_barrier_op(main_program, dummys, trainer_id)
Z
ziyoujiyi 已提交
113 114 115 116 117


@register_pass("distributed_ops_pass")
class DistributedOpsPass(PassBase):
    def __init__(self):
118
        super().__init__()
Z
ziyoujiyi 已提交
119 120 121 122 123 124 125 126 127
        self.w_2_table_id = {}
        self.emb_size = {}

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

128
    def _push_sparse_fuse(self, _program, push_sparse_ops, attrs, use_cvm_op):
Z
ziyoujiyi 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
        if attrs['use_ps_gpu']:
            return
        if len(push_sparse_ops) == 0:
            return
        show = None
        clk = None
        use_entry = False
        for param, ops in push_sparse_ops.items():
            op_first = ops[0]
            break
        if op_first.has_attr("entry"):
            entry = op_first.attr("entry")
            entry = entry.split(':')
            if len(entry) == 3 and entry[0] == 'show_click_entry':
                show_var_name = entry[1]
                click_var_name = entry[2]
145 146 147 148
                if (
                    show_var_name in _program.global_block().vars
                    and click_var_name in _program.global_block().vars
                ):
Z
ziyoujiyi 已提交
149 150 151 152 153 154 155 156 157 158 159 160
                    show = _program.global_block().vars[show_var_name]
                    clk = _program.global_block().vars[click_var_name]
                    use_entry = True
                else:
                    warnings.warn(
                        'ShowClickEntry configured, but cannot find show/click var, will not use'
                    )

        if not use_entry:
            print('ShowClickEntry not configured, will not use')
            show = _program.global_block().create_var(
                name="show",
161
                dtype=core.VarDesc.VarType.FP32,
Z
ziyoujiyi 已提交
162
                persistable=False,
163 164 165 166 167 168 169 170 171 172 173 174 175
                stop_gradient=True,
            )
            _program.global_block()._insert_op(
                index=0,
                type='fill_constant',
                inputs={},
                outputs={'Out': show},
                attrs={
                    'shape': [1],
                    'dtype': show.dtype,
                    'value': 1,
                },
            )
Z
ziyoujiyi 已提交
176 177 178

            clk = _program.global_block().create_var(
                name="clk",
179
                dtype=core.VarDesc.VarType.FP32,
Z
ziyoujiyi 已提交
180
                persistable=False,
181 182 183 184 185 186 187 188 189 190 191 192 193
                stop_gradient=True,
            )
            _program.global_block()._insert_op(
                index=0,
                type='fill_constant',
                inputs={},
                outputs={'Out': clk},
                attrs={
                    'shape': [1],
                    'dtype': clk.dtype,
                    'value': 0,
                },
            )
Z
ziyoujiyi 已提交
194 195 196 197 198 199 200 201 202 203 204 205 206

        for param, ops in push_sparse_ops.items():
            all_ops = _program.global_block().ops
            op_idxs = [all_ops.index(op) for op in ops]
            inputs = [
                _program.global_block().vars[op.input("Ids")[0]] for op in ops
            ]
            w = _program.global_block().vars[ops[0].output("W@GRAD")[0]]
            table_id = self.w_2_table_id[param]

            padding_idx = ops[0].attr("padding_idx")
            is_distributed = ops[0].attr("is_distributed")
            op_type = ops[0].type
207 208 209

            slots = [op.attr("slot") for op in ops]
            print('debug zcb slots: ', slots)
Z
ziyoujiyi 已提交
210 211 212 213 214 215 216 217
            outputs = [
                _program.global_block().vars[op.input("Out@GRAD")[0]]
                for op in ops
            ]

            for idx in op_idxs[::-1]:
                _program.global_block()._remove_op(idx)

218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
            _program.global_block().append_op(
                type="distributed_push_sparse",
                inputs={
                    "Ids": inputs,
                    'W': w,
                    "Outputs": outputs,
                    "Shows": show,
                    "Clicks": clk,
                },
                outputs={"Outputs": outputs},
                attrs={
                    "is_distributed": is_distributed,
                    "padding_idx": padding_idx,
                    "table_id": table_id,
                    "size": self.emb_size[param],
                    "use_cvm_op": use_cvm_op,
                    "slots": slots,
                },
            )
Z
ziyoujiyi 已提交
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306

    def _pull_sparse_fuse(self, _program, pull_sparse_ops, attrs, send_ctx):
        def dag_check_up_and_reorder(program, inputs, outputs):
            global_block = program.global_block()
            min_output_index = len(global_block.ops)
            max_input_index = -1
            input_indexes = [0] * len(global_block.ops)
            output_indexes = [0] * len(global_block.ops)
            for idx, op in enumerate(global_block.ops):
                for i in range(0, len(op.output_names)):
                    if input_indexes[idx] == 1:
                        break
                    outs = op.output(op.output_names[i])
                    for in_id, in_var in enumerate(inputs):
                        if in_var.name in outs:
                            input_indexes[idx] = 1
                            max_input_index = max(max_input_index, idx)
                            break

                for i in range(0, len(op.input_names)):
                    if output_indexes[idx] == 1:
                        break
                    ins = op.input(op.input_names[i])
                    for out_id, out_var in enumerate(outputs):
                        if out_var.name in ins:
                            output_indexes[idx] = 1
                            min_output_index = min(min_output_index, idx)

            for i in range(len(global_block.ops)):
                if input_indexes[i] == 1 and output_indexes[i] == 1:
                    warnings.warn(
                        "unable to re-arrange dags order to combine distributed embedding ops because a op both needs embedding table's output as input and produces ids as the same embedding table's input"
                    )
                    return

            if min_output_index < max_input_index:
                move_ops = []
                for i in range(min_output_index + 1, len(input_indexes)):
                    if input_indexes[i] == 1:
                        move_ops.append((global_block.ops[i], i))
                for i, op in enumerate(move_ops):
                    queue = list()
                    visited = set()
                    queue.append(op[1])
                    visited.add(op[0])
                    start = 0
                    while start < len(queue):
                        pos = queue[start]
                        op = global_block.ops[pos]
                        op_inputs = []
                        for k in range(0, len(op.input_names)):
                            ins = op.input(op.input_names[k])
                            op_inputs.append(ins)
                        for j in range(pos - 1, min_output_index - 1, -1):
                            op1 = global_block.ops[j]
                            if op1 in visited:
                                continue
                            found = False
                            for k in range(0, len(op1.output_names)):
                                outs = op1.output(op1.output_names[k])
                                for t in range(len(op_inputs)):
                                    for y in op_inputs[t]:
                                        if y in outs:
                                            found = True
                                            break
                                    if found:
                                        break
                                if found:
                                    break
                            if found:
307
                                if output_indexes[j]:
Z
ziyoujiyi 已提交
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
                                    warnings.warn(
                                        "unable to re-arrange dags order to combine distributed embedding ops"
                                    )
                                    return
                                queue.append(j)
                                visited.add(global_block.ops[j])
                        start = start + 1

                    queue.sort()
                    for index in queue:
                        desc = global_block.desc._insert_op(min_output_index)
                        desc.copy_from(global_block.ops[index].desc)
                        global_block.desc._remove_op(index + 1, index + 2)
                        global_block.ops[index].desc = desc
                        insert_op = global_block.ops.pop(index)
                        input_state = input_indexes.pop(index)
                        output_state = output_indexes.pop(index)
                        global_block.ops.insert(min_output_index, insert_op)
                        input_indexes.insert(min_output_index, input_state)
                        output_indexes.insert(min_output_index, output_state)
                        min_output_index = min_output_index + 1

                assert global_block.desc.op_size() == len(global_block.ops)
                for i in range(len(global_block.ops)):
                    assert global_block.desc.op(i) == global_block.ops[i].desc

334 335 336 337 338 339 340 341
        if attrs['use_ps_gpu']:
            gpups_inputs_idxs = list()
            gpups_outputs_idxs = list()
            gpups_inputs = list()
            gpups_outputs = list()
            gpups_w_size = list()
            gpups_min_distributed_idx = len(_program.global_block().ops) + 1

Z
ziyoujiyi 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
        for param, ops in pull_sparse_ops.items():
            all_ops = _program.global_block().ops
            op_device = ""
            if attrs['is_heter_ps_mode']:
                op_device = ops[0].attr("op_device")
            inputs = [
                _program.global_block().vars[op.input("Ids")[0]] for op in ops
            ]
            w = _program.global_block().vars[ops[0].input("W")[0]]
            self.emb_size[param] = w.shape[1]

            grad_name = attrs['param_name_to_grad_name'][w.name]

            table_id = -1

            for name, ctx in send_ctx.items():
                if grad_name in ctx.origin_varnames():
                    table_id = ctx.table_id()

            if table_id == -1:
                raise ValueError(
363 364
                    "can not find suitable sparse table, please check"
                )
Z
ziyoujiyi 已提交
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394

            self.w_2_table_id[param] = table_id
            padding_idx = ops[0].attr("padding_idx")
            is_distributed = ops[0].attr("is_distributed")
            op_type = ops[0].type

            outputs = [
                _program.global_block().vars[op.output("Out")[0]] for op in ops
            ]

            dag_check_up_and_reorder(_program, inputs, outputs)

            op_idxs = [all_ops.index(op) for op in ops]

            for idx in op_idxs[::-1]:
                _program.global_block()._remove_op(idx)

            inputs_idxs = [-1] * len(inputs)
            outputs_idxs = [len(_program.global_block().ops) + 1] * len(outputs)

            for idx, op in enumerate(_program.global_block().ops):
                for i in range(0, len(op.output_names)):
                    outs = op.output(op.output_names[i])
                    for in_id, in_var in enumerate(inputs):
                        if in_var.name in outs:
                            inputs_idxs[in_id] = max(idx, inputs_idxs[in_id])
                for i in range(0, len(op.input_names)):
                    ins = op.input(op.input_names[i])
                    for out_id, out_var in enumerate(outputs):
                        if out_var.name in ins:
395 396 397
                            outputs_idxs[out_id] = min(
                                idx, outputs_idxs[out_id]
                            )
Z
ziyoujiyi 已提交
398

399 400 401 402 403 404
            if attrs['use_ps_gpu']:
                gpups_inputs_idxs.extend(inputs_idxs)
                gpups_outputs_idxs.extend(outputs_idxs)
                gpups_inputs.extend(inputs)
                gpups_outputs.extend(outputs)
                gpups_w_size.extend([w.shape[1]] * len(inputs))
405 406 407
                gpups_min_distributed_idx = min(
                    min(op_idxs), gpups_min_distributed_idx
                )
408 409
                continue

Z
ziyoujiyi 已提交
410 411 412 413 414 415
            if min(outputs_idxs) - max(inputs_idxs) >= 1:
                if max(inputs_idxs) == -1:
                    distributed_idx = min(op_idxs)
                else:
                    distributed_idx = max(inputs_idxs) + 1

416 417 418
                _program.global_block()._insert_op(
                    index=distributed_idx,
                    type="distributed_lookup_table",
419
                    inputs={"Ids": inputs, 'W': w},
420 421 422 423 424 425
                    outputs={"Outputs": outputs},
                    attrs={
                        "is_distributed": is_distributed,
                        "padding_idx": padding_idx,
                        "table_id": table_id,
                        "lookup_table_version": op_type,
426 427 428
                        "op_device": op_device,
                    },
                )
Z
ziyoujiyi 已提交
429 430 431 432 433 434 435
            else:
                for i in range(len(inputs_idxs)):
                    distributed_idx = op_idxs[i]

                    _program.global_block()._insert_op(
                        index=distributed_idx,
                        type="distributed_lookup_table",
436
                        inputs={"Ids": [inputs[i]], 'W': w},
Z
ziyoujiyi 已提交
437 438 439 440 441 442
                        outputs={"Outputs": [outputs[i]]},
                        attrs={
                            "is_distributed": is_distributed,
                            "padding_idx": padding_idx,
                            "table_id": table_id,
                            "lookup_table_version": op_type,
443 444 445
                            "op_device": op_device,
                        },
                    )
Z
ziyoujiyi 已提交
446

447 448 449 450
        if attrs['use_ps_gpu'] and len(gpups_inputs) > 0:
            if max(gpups_inputs_idxs) > 0:
                raise ValueError("There can't be ops before embedding in gpups")

451 452 453 454 455 456 457 458 459 460 461 462 463
            _program.global_block()._insert_op(
                index=gpups_min_distributed_idx,
                type="pull_gpups_sparse",
                inputs={
                    "Ids": gpups_inputs,
                },
                outputs={"Out": gpups_outputs},
                attrs={
                    "size": gpups_w_size,
                    "is_distributed": True,
                    "is_sparse": True,
                },
            )
464 465 466 467 468 469 470 471 472 473 474
            PSGPU = paddle.fluid.core.PSGPU()
            try:
                gpu_slot = [int(var.name) for var in gpups_inputs]
            except (ValueError):
                raise ValueError(
                    "The slot name in gpups Should be able to convert to integer."
                )
            PSGPU.set_slot_vector(gpu_slot)
            gpu_mf_sizes = [x - 3 for x in gpups_w_size]
            PSGPU.set_slot_dim_vector(gpu_mf_sizes)

Z
ziyoujiyi 已提交
475 476 477 478 479
    def _get_pull_sparse_ops(self, _program, attrs):
        pull_sparse_ops = {}
        pull_sparse_ids = {}
        push_sparse_ops = {}
        ops = {}
480
        use_cvm_op = False
Z
ziyoujiyi 已提交
481
        for op in _program.global_block().ops:
482 483 484 485
            if (
                op.type in SPARSE_OP_TYPE_DICT.keys()
                and op.attr('remote_prefetch') is True
            ):
Z
ziyoujiyi 已提交
486
                param_name = op.input(SPARSE_OP_TYPE_DICT[op.type])[0]
Z
ziyoujiyi 已提交
487 488
                if attrs['is_heter_ps_mode'] and not attrs['is_fl_ps_mode']:
                    # TODO: trick for matchnet, need to modify for heter_ps
Z
ziyoujiyi 已提交
489
                    param_name += op.input("Ids")[0][0]
490 491
                if param_name in attrs['local_sparse']:  # for recall/ncf model
                    continue
Z
ziyoujiyi 已提交
492 493 494 495 496 497
                ops = pull_sparse_ops.get(param_name, [])
                ops.append(op)
                pull_sparse_ops[param_name] = ops
                ids = pull_sparse_ids.get(param_name, [])
                ids.append(op.input("Ids")[0])
                pull_sparse_ids[param_name] = ids
498 499 500
            if op.type == 'cvm':
                use_cvm_op = True

Z
ziyoujiyi 已提交
501 502 503
        for op in _program.global_block().ops:
            if op.type in SPARSE_GRAD_OP_TYPE_DICT.keys():
                param_name = op.input(SPARSE_GRAD_OP_TYPE_DICT[op.type])[0]
504 505 506 507
                if (
                    param_name in pull_sparse_ids
                    and op.input("Ids")[0] in pull_sparse_ids[param_name]
                ):
Z
ziyoujiyi 已提交
508 509 510 511
                    ops = push_sparse_ops.get(param_name, [])
                    ops.append(op)
                    push_sparse_ops[param_name] = ops

512
        return pull_sparse_ops, push_sparse_ops, use_cvm_op
Z
ziyoujiyi 已提交
513 514 515

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
516 517 518 519 520 521 522 523 524 525
        (
            pull_sparse_ops,
            push_sparse_ops,
            use_cvm_op,
        ) = self._get_pull_sparse_ops(main_program, attrs)
        print(
            "is_heter_ps_mode in distributed_ops_pass {}?".format(
                attrs['is_heter_ps_mode']
            )
        )
Z
ziyoujiyi 已提交
526
        send_ctx = get_the_one_send_context(
527 528
            attrs, split_dense_table=attrs['is_heter_ps_mode']
        )
Z
ziyoujiyi 已提交
529
        self._pull_sparse_fuse(main_program, pull_sparse_ops, attrs, send_ctx)
530
        self._push_sparse_fuse(main_program, push_sparse_ops, attrs, use_cvm_op)
Z
ziyoujiyi 已提交
531 532 533 534 535


@register_pass("delete_optimizer_pass")
class DeleteOptimizesPass(PassBase):
    def __init__(self):
536
        super().__init__()
Z
ziyoujiyi 已提交
537 538 539 540 541 542 543

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

544 545 546
    def _delete_optimizer_op_and_vars(
        self, _program, remote_optimize_ops, local_optimize_ops
    ):
547 548 549
        local_optimize_vars = []
        remote_optimize_vars = []
        remote_optimize_op_role_vars = []
Z
ziyoujiyi 已提交
550 551
        optimize_need_delete_vars = []

552 553 554 555 556 557 558 559
        for op in local_optimize_ops:
            local_optimize_vars.extend(op.input_arg_names)

        for op in remote_optimize_ops:
            remote_optimize_vars.extend(op.input_arg_names)
            remote_optimize_op_role_vars.extend(op.attr("op_role_var"))

        remote_optimize_vars = list(
560 561
            set(remote_optimize_vars)
        )  # param + grad + optimizer_state + learning_rate
562
        remote_optimize_op_role_vars = list(
563 564
            set(remote_optimize_op_role_vars)
        )  # param + grad
565
        print(
566 567 568 569 570 571
            "remote_optimize_vars: {}, remote_optimize_op_role_vars: {}, local_optimize_vars: {}".format(
                remote_optimize_vars,
                remote_optimize_op_role_vars,
                local_optimize_vars,
            )
        )
572 573 574 575
        for var in remote_optimize_vars:
            if var in local_optimize_vars:
                continue
            if var not in remote_optimize_op_role_vars:
Z
ziyoujiyi 已提交
576 577 578
                optimize_need_delete_vars.append(var)
        need_delete_optimize_vars = list(set(optimize_need_delete_vars))

579
        delete_ops(_program.global_block(), remote_optimize_ops)
Z
ziyoujiyi 已提交
580 581 582 583 584 585
        for var in need_delete_optimize_vars:
            if _program.global_block().has_var(var):
                _program.global_block()._remove_var(var)

    def _add_lr_var(self, main_program, attrs):
        # Todo: hard code for pe
586 587 588 589 590 591 592 593 594 595 596
        lr_var = (
            attrs['origin_main_program'].global_block().vars["learning_rate_0"]
        )
        main_program.global_block().create_var(
            name=lr_var.name,
            shape=lr_var.shape,
            dtype=lr_var.dtype,
            type=lr_var.type,
            lod_level=lr_var.lod_level,
            persistable=True,
        )
Z
ziyoujiyi 已提交
597 598 599

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
600
        all_optimize_ops = get_optimize_ops(main_program)
601 602 603
        remote_optimize_ops = get_optimize_ops(
            main_program, attrs['remote_sparse']
        )
Z
ziyoujiyi 已提交
604
        lr_ops = get_lr_ops(main_program)
605 606
        remote_optimize_ops.extend(lr_ops)
        local_optimize_ops = list(
607 608 609 610 611
            set(all_optimize_ops) - set(remote_optimize_ops)
        )
        self._delete_optimizer_op_and_vars(
            main_program, remote_optimize_ops, local_optimize_ops
        )
Z
ziyoujiyi 已提交
612 613 614 615 616 617 618 619

        if hasattr(attrs['origin_main_program'], 'lr_sheduler'):
            self._add_lr_var(main_program, attrs)


@register_pass("delete_extra_optimizer_pass")
class DeleteExtraOptimizerPass(PassBase):
    def __init__(self):
620
        super().__init__()
Z
ziyoujiyi 已提交
621 622 623 624 625 626 627 628 629

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
630 631
        remote_optimize_vars = []
        remote_optimize_op_role_vars = []
Z
ziyoujiyi 已提交
632
        optimize_need_delete_vars = []
633
        all_optimize_ops = get_optimize_ops(main_program)
634 635 636
        remote_optimize_ops = get_optimize_ops(
            main_program, attrs['remote_sparse']
        )
637
        local_optimize_ops = list(
638 639
            set(all_optimize_ops) - set(remote_optimize_ops)
        )
640 641 642 643 644 645 646 647 648 649 650 651 652 653

        local_optimize_vars = []
        for op in local_optimize_ops:
            local_optimize_vars.extend(op.input_arg_names)

        for op in remote_optimize_ops:
            remote_optimize_vars.extend(op.input_arg_names)
            remote_optimize_op_role_vars.extend(op.attr("op_role_var"))

        remote_optimize_vars = list(set(remote_optimize_vars))
        remote_optimize_op_role_vars = list(set(remote_optimize_op_role_vars))
        for var in remote_optimize_vars:
            if var in local_optimize_vars:
                continue
Z
ziyoujiyi 已提交
654 655
            if 'learning_rate_0' == var:
                continue
656
            if var not in remote_optimize_op_role_vars:
Z
ziyoujiyi 已提交
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
                optimize_need_delete_vars.append(var)
        need_delete_optimize_vars = list(set(optimize_need_delete_vars))

        init_ops = []
        for var in need_delete_optimize_vars:
            param_init_op = []
            for op in startup_program.global_block().ops:
                if var in op.output_arg_names:
                    param_init_op.append(op)
            init_ops.extend(param_init_op)
        delete_ops(startup_program.global_block(), init_ops)

        for var in need_delete_optimize_vars:
            if startup_program.global_block().has_var(var):
                startup_program.global_block()._remove_var(var)


@register_pass("fake_init_ops_pass")
class FakeInitOpsPass(PassBase):
    def __init__(self):
677
        super().__init__()
Z
ziyoujiyi 已提交
678 679 680 681 682 683 684 685

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _get_sparse_table_names(self, attrs):
686 687 688 689 690 691
        dist_varnames = get_sparse_tablenames(
            attrs['origin_main_programs'], True
        )
        sparse_varnames = get_sparse_tablenames(
            attrs['origin_main_programs'], False
        )
Z
ziyoujiyi 已提交
692 693
        return list(set(dist_varnames + sparse_varnames))

694 695 696
    def _fake_init_sparsetable(
        self, startup_program, sparse_table_names, attrs
    ):
Z
ziyoujiyi 已提交
697 698
        # delete table init op
        for table_name in sparse_table_names:
699
            table_var = startup_program.global_block().vars[table_name]
700 701 702 703
            if (
                str(table_var).split(":")[0].strip().split()[-1]
                in attrs['local_sparse']
            ):
704
                continue
Z
ziyoujiyi 已提交
705
            table_param_init_op = []
706
            for op in startup_program.global_block().ops:
Z
ziyoujiyi 已提交
707 708 709 710
                if table_name in op.output_arg_names:
                    table_param_init_op.append(op)
            init_op_num = len(table_param_init_op)
            if init_op_num != 1:
711 712 713
                raise ValueError(
                    "table init op num should be 1, now is " + str(init_op_num)
                )
Z
ziyoujiyi 已提交
714
            table_init_op = table_param_init_op[0]
715
            startup_program.global_block().append_op(
Z
ziyoujiyi 已提交
716 717 718
                type="fake_init",
                inputs={},
                outputs={"Out": table_var},
719 720
                attrs={"shape": table_init_op.attr('shape')},
            )
721
            delete_ops(startup_program.global_block(), table_param_init_op)
Z
ziyoujiyi 已提交
722 723 724 725

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        sparse_tables = self._get_sparse_table_names(attrs)
726
        self._fake_init_sparsetable(startup_program, sparse_tables, attrs)
Z
ziyoujiyi 已提交
727 728 729 730 731


@register_pass("ps_gpu_pass")
class PsGpuPass(PassBase):
    def __init__(self):
732
        super().__init__()
Z
ziyoujiyi 已提交
733 734 735 736 737 738 739 740

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _add_push_box_sparse_op(self, program):
D
danleifeng 已提交
741 742 743 744
        insert_index = -1
        for idx, op in list(enumerate(program.global_block().ops)):
            if op.type == "lookup_table_grad":
                insert_index = idx
Z
ziyoujiyi 已提交
745
        for op in program.global_block().ops:
D
danleifeng 已提交
746
            if op.type != "pull_box_sparse" and op.type != "pull_gpups_sparse":
Z
ziyoujiyi 已提交
747 748
                continue
            grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
749 750
                op.desc, set(), []
            )
Z
ziyoujiyi 已提交
751
            for op_desc in grad_op_desc:
D
danleifeng 已提交
752
                new_op_desc = program.global_block().desc._insert_op(
753 754
                    insert_index + 1
                )
Z
ziyoujiyi 已提交
755 756
                new_op_desc.copy_from(op_desc)
                new_op_desc._set_attr(op_role_attr_name, backward)
757
                new_op = paddle.fluid.framework.Operator(
758 759
                    program.global_block(), new_op_desc
                )
D
danleifeng 已提交
760 761
                program.global_block().ops.insert(insert_index + 1, new_op)
                program.global_block()._sync_with_cpp()
Z
ziyoujiyi 已提交
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808

    def _remove_optimizer_var(self, program):
        embedding_w = {}
        for idx, op in list(enumerate(program.global_block().ops)):
            if op.type == "lookup_table_grad":
                for name in op.input("W"):
                    embedding_w[name] = 1

        optimize_vars = []
        optimize_op_role_vars = []
        optimize_need_delete_vars = []
        for op in get_optimize_ops(program):
            for name in op.input("Param"):
                if name in embedding_w:
                    optimize_op_role_vars.extend(op.attr("op_role_var"))
                    for key_name in op.input_names:
                        if key_name == "LearningRate":
                            continue
                        for var in op.input(key_name):
                            optimize_vars.append(var)

        optimize_vars = list(set(optimize_vars))
        optimize_op_role_vars = list(set(optimize_op_role_vars))

        for var in optimize_vars:
            if var not in optimize_op_role_vars:
                optimize_need_delete_vars.append(var)
        need_delete_optimize_vars = list(set(optimize_need_delete_vars))

        for name in need_delete_optimize_vars:
            if program.global_block().has_var(name):
                program.global_block()._remove_var(name)

    def _remove_lookup_table_grad_op_and_var(self, program):
        lookup_table_grad_var = {}
        remove_op_index = []
        remove_var = []
        for idx, op in list(enumerate(program.global_block().ops)):
            if op.type == "lookup_table_grad":
                for name in op.output("W@GRAD"):
                    lookup_table_grad_var[name] = 1
                    remove_op_index.append(idx)
                    remove_var.append(name)
                for name in op.input("W"):
                    lookup_table_grad_var[name] = 1

        for idx, op in list(enumerate(program.global_block().ops)):
809
            if op.type == "pull_box_sparse" or op.type == "pull_gpups_sparse":
Z
ziyoujiyi 已提交
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
                continue
            for key_name in op.input_names:
                for var in op.input(key_name):
                    if var in lookup_table_grad_var:
                        remove_op_index.append(idx)
                        break

        remove_op_index = list(set(remove_op_index))
        remove_op_index.sort(reverse=True)
        for idx in remove_op_index:
            program.global_block()._remove_op(idx)
        for name in remove_var:
            program.global_block()._remove_var(name)

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        self._add_push_box_sparse_op(main_program)
        self._remove_optimizer_var(main_program)
        self._remove_lookup_table_grad_op_and_var(main_program)


@register_pass("ps_transpile_pass")
class PsTranspilePass(PassBase):
    def __init__(self):
834
        super().__init__()
Z
ziyoujiyi 已提交
835 836 837 838 839 840 841 842 843 844 845

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        t = SingleProcessMultiThread()
        env = get_dist_env()
846 847 848 849 850 851 852 853
        t.transpile(
            startup_program=startup_program,
            main_program=main_program,
            rank=env["trainer_id"],
            endpoints=env["trainer_endpoints"],
            current_endpoint=env['current_endpoint'],
            wait_port=False,
        )
Z
ziyoujiyi 已提交
854 855 856 857 858


@register_pass("split_heter_worker_ops_pass")
class SplitHeterWorkerOpsPass(PassBase):
    def __init__(self):
859
        super().__init__()
Z
ziyoujiyi 已提交
860 861 862 863 864 865 866

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

867 868 869 870 871 872 873 874 875
    def _create_heter_program(
        self,
        program,
        attrs,
        heter_program,
        program_block_ops_list,
        heter_ops,
        block_var_detail,
    ):
Z
ziyoujiyi 已提交
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
        # This function mainly includes the following contents:
        # 1. For every heter block:
        #     a) copy heter device op from origin program
        #     b) create variables which belong to heter op:
        #         -> if variable is persistable, clone it in global_scope
        #         -> if variable is temp, create it in heter block
        #     c) create communicate related op as follow:
        #         joint_var.0_1 -> slice -> reshape -> origin_var
        #         origin_var -> origin_program
        #         reshape -> concat -> joint_var.1_2
        #     d) copy send op from origin program for var@grad which loacted in current heter block
        #     e) re-check every op in current blcok if its device is not current heter devie
        # 2. Create send op for step counter in last heter-block
        # 3. Create Listen&Serv OP and Send&Recv OP for distributed training
        # 4. update CompileTimeStrategy for heter_program

        optimizer_block = []
        grad_to_block_id = []
        send_grad_var_list = []

        pre_block_idx = heter_program.num_blocks - 1
        role_maker = attrs['role_maker']
        current_device = role_maker._heter_device_type().lower()
        stage_id = int(role_maker._get_stage_id())

901 902 903 904 905 906
        heter_block_ops_forward = program_block_ops_list[stage_id - 1][
            "forward"
        ]
        heter_block_ops_backward = program_block_ops_list[stage_id - 1][
            "backward"
        ]
Z
ziyoujiyi 已提交
907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927

        heter_block = heter_program._create_block(pre_block_idx)
        optimizer_block.append(heter_block)
        for _, op in enumerate(heter_block_ops_forward):
            block_append_op(heter_program, program, heter_block, op)

        entrance_vars = block_var_detail[stage_id - 1]["forward"]["entrance"]
        add_vars_by_var_list(entrance_vars, program, heter_program, heter_block)
        exit_vars = block_var_detail[stage_id - 1]["forward"]["exit"]
        add_vars_by_var_list(exit_vars, program, heter_program, heter_block)

        first_op_index_fp = len(heter_block.ops)

        if stage_id < len(program_block_ops_list):

            heter_block_bp = heter_program._create_block(pre_block_idx)
            optimizer_block.append(heter_block_bp)

            for _, op in enumerate(heter_block_ops_backward):
                block_append_op(heter_program, program, heter_block_bp, op)

928 929 930 931 932 933
            bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][
                "entrance"
            ]
            add_vars_by_var_list(
                bp_entrance_vars, program, heter_program, heter_block_bp
            )
Z
ziyoujiyi 已提交
934
            bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"]
935 936 937 938 939 940
            add_vars_by_var_list(
                bp_exit_vars, program, heter_program, heter_block_bp
            )
            backward_comm_info = get_communicate_var_info(
                program, stage_id, bp_entrance_vars, type="backward"
            )
Z
ziyoujiyi 已提交
941

942 943 944 945 946
            grad_to_block_id.append(
                backward_comm_info["block_input_var_name"]
                + ":"
                + str(heter_block_bp.idx)
            )
Z
ziyoujiyi 已提交
947 948 949 950 951

        else:
            for _, op in enumerate(heter_block_ops_backward):
                block_append_op(heter_program, program, heter_block, op)

952 953 954 955 956 957
            bp_entrance_vars = block_var_detail[stage_id - 1]["backward"][
                "entrance"
            ]
            add_vars_by_var_list(
                bp_entrance_vars, program, heter_program, heter_block
            )
Z
ziyoujiyi 已提交
958
            bp_exit_vars = block_var_detail[stage_id - 1]["backward"]["exit"]
959 960 961
            add_vars_by_var_list(
                bp_exit_vars, program, heter_program, heter_block
            )
Z
ziyoujiyi 已提交
962 963 964

            heter_block_bp = heter_block

965 966 967
        forward_comm_info = get_communicate_var_info(
            program, stage_id, entrance_vars, type="forward"
        )
Z
ziyoujiyi 已提交
968

969 970 971 972 973
        grad_to_block_id.append(
            forward_comm_info["block_input_var_name"]
            + ":"
            + str(heter_block.idx)
        )
Z
ziyoujiyi 已提交
974 975 976 977

        first_op_index_bp = len(heter_block_bp.ops)

        if stage_id <= len(block_var_detail) - 1:
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
            static_var = insert_communicate_op(
                program,
                role_maker,
                heter_block,
                stage_id,
                first_op_index_fp,
                block_var_detail,
                current_device,
            )
        static_var_bp = insert_communicate_op(
            program,
            role_maker,
            heter_block_bp,
            stage_id,
            first_op_index_bp,
            block_var_detail,
            current_device,
            False,
        )
Z
ziyoujiyi 已提交
997 998

        # add send op
999
        send_grad_var_list = add_send_op(
1000 1001 1002 1003
            program,
            heter_block_bp,
            block_var_detail[stage_id - 1]["backward"]["persistables"],
        )
Z
ziyoujiyi 已提交
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017

        # add step conter
        send_input_vars = []
        dummy_output = []
        pserver_endpoints = get_ps_endpoints(role_maker)
        attrs = {
            "message_to_block_id": grad_to_block_id,
            "optimize_blocks": optimizer_block,
            # runtime attribute
            "endpoint": get_heter_worker_endpoint(role_maker),
            "fanin": len(get_previous_stage_trainers(role_maker)),
            "pserver_id": get_role_id(role_maker),
            "distributed_mode": attrs['ps_mode'],
            "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)),
1018
            RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
Z
ziyoujiyi 已提交
1019 1020
        }
        # append the listen_and_serv op
1021 1022 1023 1024 1025 1026
        heter_program.global_block().append_op(
            type="heter_listen_and_serv",
            inputs={'X': []},
            outputs={},
            attrs=attrs,
        )
Z
ziyoujiyi 已提交
1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
        # TODO check heter program

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        """
        split heter worker program from origin-program
        1. find heter op (located on different device)
        2. find input&output of every heter-block
        3. create heter worker program, add listen&serv op
        """
        attrs = pass_ctx._attrs
        default_deveice = "cpu"
        program, heter_ops, _, program_block_ops = find_heter_ops(
1039 1040
            main_program, default_deveice
        )
Z
ziyoujiyi 已提交
1041 1042 1043 1044 1045 1046 1047 1048
        if len(heter_ops) == 0:
            warnings.warn(
                "Currently running in Heter Parameter Server mode, but no OP running on heterogeneous devices, Please check your code."
            )
            main_program = program
            return

        program_block_ops = union_forward_gradient_op(program_block_ops)
1049 1050 1051
        block_vars_detail = find_block_joints(
            program, program_block_ops, heter_ops
        )
Z
ziyoujiyi 已提交
1052
        heter_program = framework.Program()
1053 1054 1055 1056 1057 1058 1059 1060
        self._create_heter_program(
            program,
            attrs,
            heter_program,
            program_block_ops,
            heter_ops,
            block_vars_detail,
        )
Z
ziyoujiyi 已提交
1061 1062 1063 1064 1065 1066
        main_program = heter_program


@register_pass("split_trainer_ops_pass")
class SplitTrainerOpsPass(PassBase):
    def __init__(self):
1067
        super().__init__()
Z
ziyoujiyi 已提交
1068 1069 1070 1071 1072 1073 1074

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

1075 1076 1077
    def _replace_ops_by_communicate_op(
        self, program, attrs, heter_block_index, ops_list, block_var_detail
    ):
1078 1079 1080 1081 1082 1083 1084 1085
        all_op = program.global_block().ops
        start_op = ops_list[0]
        first_op_idx = -1
        for op in all_op:
            if str(op) == str(start_op):
                first_op_idx = all_op.index(op)
                break
        assert first_op_idx != -1
1086
        delete_same_ops(program.global_block(), ops_list)
1087 1088 1089 1090 1091 1092 1093

        entrance_var = []
        role_maker = attrs['role_maker']
        if heter_block_index == 1:
            next_heter_worker_endpoints = get_next_stage_trainers(role_maker)

            entrance_var = block_var_detail[heter_block_index]["forward"][
1094 1095
                "entrance"
            ]
1096

1097 1098 1099
            comm_info = get_communicate_var_info(
                program, heter_block_index + 1, entrance_var
            )
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
            program.global_block()._insert_op(
                index=first_op_idx,
                type="send_and_recv",
                inputs={"X": program.global_block().vars[entrance_var[0]]},
                outputs={"Out": []},
                attrs={
                    "mode": "forward",
                    "send_var_name": entrance_var + ["microbatch_id"],
                    "recv_var_name": [],
                    "message_name": comm_info["block_input_var_name"],
                    "next_endpoints": next_heter_worker_endpoints,
                    "previous_endpoints": [],
                    "trainer_id": get_role_id(role_maker),
1113 1114 1115
                    RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
                },
            )
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139

        return entrance_var

    def _remove_var_pair_by_grad(self, var_name, attrs):
        for index, pair in enumerate(attrs['merged_variables_pairs']):
            var = pair[0]
            var_grad = pair[1]
            if var_grad.merged_var.name == var_name:
                del attrs['merged_variables_pairs'][index]

        for index, pair in enumerate(attrs['merged_dense_pairs']):
            var = pair[0]
            var_grad = pair[1]
            if var_grad.merged_var.name == var_name:
                del attrs['merged_dense_pairs'][index]
                return

        for index, pair in enumerate(attrs['merged_sparse_pairs']):
            var = pair[0]
            var_grad = pair[1]
            if var_grad.merged_var.name == var_name:
                del attrs['merged_sparse_pairs'][index]
                return

1140 1141 1142
    def _remove_trainer_send_op(
        self, program, attrs, heter_block_index, block_var_detail
    ):
1143 1144 1145
        # if trainer do FF->BP->SEND, it has follow vars: var, var@GRAD
        # if trainer only do SEND, it has one var: var@GRAD
        # Delete Send op ,if trainer doesn't has pair var (var<->var@GRAD)
1146 1147 1148 1149
        persistables = (
            block_var_detail[heter_block_index]["forward"]["persistables"]
            + block_var_detail[heter_block_index]["backward"]["persistables"]
        )
1150 1151 1152
        need_remove_send_op = []
        need_remove_grad_var = []
        for op in find_send_op(program):
1153 1154 1155
            input_list, _ = find_op_input_output(
                program, program.global_block(), op
            )
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
            for var_name in input_list:
                origin_var_name = var_name.split("@GRAD")[0]
                if origin_var_name in persistables:
                    need_remove_send_op.append(op)
                    need_remove_grad_var.append(var_name)
        need_remove_send_op = list(set(need_remove_send_op))
        delete_ops(program.global_block(), need_remove_send_op)
        for grad_var_name in need_remove_grad_var:
            self._remove_var_pair_by_grad(grad_var_name, attrs)

1166 1167 1168 1169 1170 1171 1172 1173
    def _create_trainer_program(
        self,
        program,
        origin_program,
        attrs,
        program_block_ops_list,
        block_var_detail,
    ):
Z
ziyoujiyi 已提交
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
        # This function mainly includes the following contents:
        # 1. For every heter block in origin program
        #     a) delete heter op and related variables
        #     b) add send&recv op
        #     c) add communicate ops as follows:
        #         origin_var -> reshape -> concat -> joint_var.0_1
        #         send&recv op(send joint_var.0_1; recv joint_var.1_2)
        #         joint_var.1_2 -> slice -> reshape -> origin_var
        #     d) remove send op which related var@grad is not in trainer program
        # 2. check every op's device
        static_var = []
        for heter_block_index in range(1, len(program_block_ops_list)):
1186 1187 1188 1189
            ops_list = (
                program_block_ops_list[heter_block_index]["forward"]
                + program_block_ops_list[heter_block_index]["backward"]
            )
1190
            static_var += self._replace_ops_by_communicate_op(
1191 1192 1193 1194 1195
                program, attrs, heter_block_index, ops_list, block_var_detail
            )
            self._remove_trainer_send_op(
                program, attrs, heter_block_index, block_var_detail
            )
Z
ziyoujiyi 已提交
1196 1197 1198 1199 1200

        optimizer_block = []
        grad_to_block_id = []

        bp_ops_list = program_block_ops_list[0]["backward"]
1201
        delete_same_ops(program.global_block(), bp_ops_list)
1202
        delete_trainer_useless_var(program, static_var)
1203 1204 1205
        backward_block = create_backward_block(
            program, origin_program, bp_ops_list, block_var_detail
        )
Z
ziyoujiyi 已提交
1206 1207

        bp_entrance_vars = block_var_detail[0]["backward"]["entrance"]
1208 1209 1210 1211 1212 1213 1214 1215 1216
        backward_comm_info = get_communicate_var_info(
            origin_program, 1, bp_entrance_vars, type="backward"
        )

        grad_to_block_id.append(
            backward_comm_info["block_input_var_name"]
            + ":"
            + str(backward_block.idx)
        )
Z
ziyoujiyi 已提交
1217 1218 1219 1220 1221 1222
        optimizer_block.append(backward_block)
        role_maker = attrs['role_maker']
        attrs = {
            "message_to_block_id": grad_to_block_id,
            "optimize_blocks": optimizer_block,
            # runtime attribute
1223 1224
            "endpoint": get_trainer_endpoint(
                role_maker
1225 1226
            ),  # get trainer endpoint
            "fanin": 0,  # get heter worker
Z
ziyoujiyi 已提交
1227 1228 1229
            "pserver_id": get_role_id(role_maker),
            "distributed_mode": attrs['ps_mode'],
            "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)),
1230
            RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
Z
ziyoujiyi 已提交
1231 1232
        }
        # append the listen_and_serv op
1233 1234 1235 1236 1237 1238 1239
        program.global_block()._insert_op(
            index=0,
            type="heter_listen_and_serv",
            inputs={'X': []},
            outputs={},
            attrs=attrs,
        )
Z
ziyoujiyi 已提交
1240 1241

        ## TODO add check for bp block
1242
        # check_op_device(program.global_block(), DEFAULT_DEVICE)
Z
ziyoujiyi 已提交
1243 1244 1245 1246 1247 1248

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        """
        split cpu-trainer program from origin-program
        1. find heter op (located on different device)
        2. find input&output of every heter-block
1249
        3. create cpu-trainer program, add send&recv op
Z
ziyoujiyi 已提交
1250 1251 1252 1253
        """
        attrs = pass_ctx._attrs
        default_device_ = 'cpu'
        program, heter_ops, default_ops, program_block_ops = find_heter_ops(
1254 1255
            main_program, default_device_
        )
Z
ziyoujiyi 已提交
1256 1257
        program_block_ops = union_forward_gradient_op(program_block_ops)

1258 1259 1260
        block_vars_detail = find_block_joints(
            program, program_block_ops, heter_ops
        )
Z
ziyoujiyi 已提交
1261
        trainer_program = program.clone()
1262 1263 1264 1265 1266 1267 1268
        self._create_trainer_program(
            trainer_program,
            program,
            attrs,
            program_block_ops,
            block_vars_detail,
        )
Z
ziyoujiyi 已提交
1269 1270 1271 1272 1273 1274
        main_program = trainer_program


@register_pass("set_heter_pipeline_opt_pass")
class SetHeterPipelineOptPass(PassBase):
    def __init__(self):
1275
        super().__init__()
Z
ziyoujiyi 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        role_maker = attrs['role_maker']
        num_microbatches = attrs['user_defined_strategy'].pipeline_configs[
1287 1288
            'accumulate_steps'
        ]
Z
ziyoujiyi 已提交
1289

1290
        startup_program._heter_pipeline_opt = {
Z
ziyoujiyi 已提交
1291 1292 1293
            "startup_program": startup_program,
            "pipeline_stage": int(role_maker._get_stage_id()) - 1,
            "heter_place": role_maker._heter_device(),
1294
            "is_fl_mode": 1,
Z
ziyoujiyi 已提交
1295
        }
1296
        main_program._heter_pipeline_opt = {
Z
ziyoujiyi 已提交
1297 1298
            "trainer": "HeterPipelineTrainer",
            "device_worker": "HeterSection",
1299
            "trainers": role_maker._get_stage_trainers(),  # trainer num in each stage
Z
ziyoujiyi 已提交
1300 1301 1302 1303 1304 1305
            "trainer_id": int(role_maker._role_id()),
            "pipeline_stage": int(role_maker._get_stage_id()) - 1,
            "num_pipeline_stages": int(role_maker._get_num_stage()),
            "section_program": main_program,
            "num_microbatches": num_microbatches,
            "heter_place": role_maker._heter_device(),
1306
            "is_fl_mode": 1,
Z
ziyoujiyi 已提交
1307
        }
1308 1309 1310 1311 1312


@register_pass("split_fl_ops_pass")
class SplitFlOpsPass(PassBase):
    def __init__(self):
1313
        super().__init__()
1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343
        self.PART_A_DEVICE_FlAG = 'gpu:0'
        self.PART_A_JOINT_OP_DEVICE_FlAG = 'gpu:2'
        self.PART_B_DEVICE_FlAG = 'gpu:1'
        self.PART_B_JOINT_OP_DEVICE_FlAG = 'gpu:3'

    def _check_self(self):
        return True

    def _check_conflict(self, other_pass):
        return True

    def _insert_encrypt_op(self):
        pass

    def _insert_decrypt_op(self):
        pass

    def _clear_op_device_flag(self, program):
        for block in program.blocks:
            for op in block.ops:
                device = op.attr(OP_DEVICE_KEY)
                op._set_attr(OP_DEVICE_KEY, '') if device != '' else None

    def _split_fl_program(self):
        self.partA_ops = []
        self.partB_ops = []
        party_program_map = defaultdict(Program)
        block = self.ori_main_program.block(0)
        for op in block.ops:
            device = op.attr(OP_DEVICE_KEY)
1344 1345 1346 1347 1348
            if (
                device == self.PART_A_DEVICE_FlAG
                or device == ''
                or device == self.PART_A_JOINT_OP_DEVICE_FlAG
            ):
1349 1350
                program = party_program_map['a']
                self.partA_ops.append(op)
1351 1352 1353 1354
            elif (
                device == self.PART_B_DEVICE_FlAG
                or device == self.PART_B_JOINT_OP_DEVICE_FlAG
            ):
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376
                program = party_program_map['b']
                self.partB_ops.append(op)
            op_desc = op.desc
            ap_op = program.global_block().desc.append_op()
            ap_op.copy_from(op_desc)
            ap_op._set_attr(OP_DEVICE_KEY, device)

        for key in ['a', 'b']:
            program = party_program_map[key]
            program._sync_with_cpp()

        return party_program_map

    def _insert_partA_communicate_op(self, block, idx):
        comm_info = "forward_joint_{}_{}@fl_ps".format(1, 2)
        block._insert_op(
            idx,
            type='send_and_recv',
            inputs={'X': self.partA_to_partB_tensor},
            outputs={'Out': []},
            attrs={
                'mode': 'forward',  # mode 直接关联前向和反向 channel 选择
1377 1378
                'send_var_name': self.partA_to_partB_tensor_name
                + ["microbatch_id"],
1379 1380
                'recv_var_name': [],
                'message_name': comm_info,
1381 1382 1383 1384 1385 1386
                'next_endpoints': get_next_stage_trainers(
                    self.role_maker
                ),  # partB_endpoints
                'previous_endpoints': get_previous_stage_trainers(
                    self.role_maker
                ),
1387
                'trainer_id': get_role_id(self.role_maker),  # global id
1388 1389 1390
                RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
            },
        )
1391 1392 1393
        return

    def _insert_partB_communicate_op(self, block, idx):
1394
        comm_info = "backward_joint_{}_{}@fl_ps".format(2, 1)
1395 1396 1397 1398 1399 1400 1401
        block._insert_op(
            idx,
            type='send_and_recv',
            inputs={'X': self.partB_to_partA_grad},
            outputs={'Out': []},
            attrs={
                'mode': 'backward',
1402 1403
                'send_var_name': self.partB_to_partA_grad_name
                + ["microbatch_id"],
1404 1405
                'recv_var_name': [],
                'message_name': comm_info,
1406 1407 1408 1409 1410 1411
                'next_endpoints': get_next_stage_trainers(
                    self.role_maker
                ),  # partA_endpoints
                'previous_endpoints': get_previous_stage_trainers(
                    self.role_maker
                ),
1412
                'trainer_id': get_role_id(self.role_maker),  # global id
1413 1414 1415
                RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
            },
        )
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433
        return

    def _create_var_for_block(self, vars, block):
        for var in vars:
            if block._find_var_recursive(str(var)):
                continue
            source_var = self.ori_main_block._var_recursive(str(var))
            if isinstance(var, Parameter):
                dest_var = block.create_parameter(
                    name=source_var.name,
                    shape=source_var.shape,
                    dtype=source_var.dtype,
                    type=source_var.type,
                    lod_level=source_var.lod_level,
                    stop_gradient=source_var.stop_gradient,
                    trainable=source_var.trainable,
                    optimize_attr=source_var.optimize_attr,
                    regularizer=source_var.regularizer,
1434 1435
                    error_clip=source_var.error_clip,
                )
1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
            else:
                dest_var = block._clone_variable(source_var, False)
            dest_var.stop_gradient = source_var.stop_gradient
            if hasattr(source_var, 'is_distributed'):
                dest_var.is_distributed = source_var.is_distributed

    def _get_block_by_idx(self, op_list, program, block_idx):
        if block_idx < len(program.blocks):
            new_block = program.block(block_idx)
        else:
            new_block = program._create_block()
        for _, op in enumerate(op_list):
            ap_op = new_block.desc.append_op()
            ap_op.copy_from(op.desc)
            ap_op._set_attr(OP_DEVICE_KEY, op.attr(OP_DEVICE_KEY))
            vars = op.desc.input_arg_names() + op.desc.output_arg_names()
            self._create_var_for_block(vars, new_block)
        new_block._sync_with_cpp()
        return new_block

    def _find_joint_forward_op(self, block, flag):
        op_idx = 0
        for op in block.ops:
            if is_forward_op(op) and op.attr(OP_DEVICE_KEY) == flag:
                return op_idx
            else:
                op_idx += 1
        return op_idx

    def _find_joint_backward_op(self, block, flag):
        op_idx = 0
        for op in block.ops:
            if is_backward_op(op) and op.attr(OP_DEVICE_KEY) == flag:
                return op_idx
            else:
                op_idx += 1
        return op_idx

    def _get_partB_to_partA_grad(self, block, flag):
        op_idx = self._find_joint_backward_op(block, flag)
        op = block.ops[op_idx]
        vars1 = op.desc.input_arg_names()
        op_idx = self._find_joint_forward_op(block, flag)
        op = block.ops[op_idx]
        vars2 = op.desc.output_arg_names()
        self.partB_to_partA_grad_name = list(set(vars1) - set(vars2))
        self.partB_to_partA_grad = []
        for var_name in self.partB_to_partA_grad_name:
            self.partB_to_partA_grad.append(self.ori_main_block.var(var_name))

    def _find_dense_grad_vars(self, bp_op_list):
        program = self.ori_main_program
1488
        bp_op_input, bp_op_output = find_ops_list_input_output(
1489 1490 1491 1492 1493
            program, bp_op_list
        )
        return screen_persistables(program, bp_op_input) + screen_persistables(
            program, bp_op_output
        )
1494 1495 1496 1497

    def _get_partA_program(self, block):
        # 1. create block 0
        # 1.1 insert send op
1498 1499 1500
        op_idx = self._find_joint_forward_op(
            block, self.PART_A_JOINT_OP_DEVICE_FlAG
        )
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
        op_list = []
        for i in range(len(block.ops)):
            op = block.ops[i]
            op_list.append(op)
            if i == op_idx:
                out_name = op.desc.output_arg_names()[0]
                self.partA_to_partB_tensor_name = op.desc.output_arg_names()
                self.partA_to_partB_tensor = self.ori_main_block.var(out_name)
                break
        first_block = self._get_block_by_idx(op_list, self.partA_program, 0)
        self._insert_partA_communicate_op(first_block, op_idx + 1)
        # logger.info('partA-first_block:{}'.format(first_block))

        # 2. create block 1
        bp_op_list = get_bp_op_list(block)
        push_sparse_op_list = get_distributed_push_sparse_op_list(block)
        # logger.info('bp_op_list: {}'.format(bp_op_list))
1518 1519 1520
        second_block = self._get_block_by_idx(
            bp_op_list + push_sparse_op_list, self.partA_program, 1
        )
1521
        # 2.1. insert partA recv op
1522 1523 1524 1525 1526
        block_input_flag = "backward_joint_{}_{}@fl_ps".format(2, 1)
        grad_to_block_id = block_input_flag + ":" + str(second_block.idx)
        attrs = {
            "message_to_block_id": [grad_to_block_id],
            "optimize_blocks": [second_block],
1527
            "endpoint": get_trainer_endpoint(self.role_maker),
1528 1529 1530 1531
            "fanin": 0,
            "pserver_id": get_role_id(self.role_maker),
            "distributed_mode": self.ps_mode,
            "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)),
1532
            RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
1533
        }
1534 1535 1536 1537 1538 1539 1540
        second_block._insert_op(
            index=0,
            type='heter_listen_and_serv',
            inputs={'X': []},
            outputs={},
            attrs=attrs,
        )
1541 1542 1543 1544 1545 1546 1547 1548 1549
        # 2.2 insert push dense grad op
        send_ops = find_send_op(self.ori_main_program)  # push dense
        delete_same_ops(block, send_ops)
        dense_grad_vars = self._find_dense_grad_vars(bp_op_list)
        add_send_op(self.ori_main_program, second_block, dense_grad_vars)
        # logger.info('partA-second_block:{}'.format(second_block))

    def _get_partB_program(self, block):
        op_idx1 = self._find_joint_forward_op(
1550 1551 1552 1553 1554
            block, self.PART_B_JOINT_OP_DEVICE_FlAG
        )  # elementwise_add op
        op_idx2 = self._find_joint_backward_op(
            block, self.PART_B_JOINT_OP_DEVICE_FlAG
        )
1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
        op_cnt = 0
        op_list1 = []
        op_list2 = []
        op_list3 = []
        for op in block.ops:
            if op_cnt < op_idx1:
                op_list1.append(op)
            elif op_cnt <= op_idx2:
                op_list2.append(op)
            else:
                op_list3.append(op)
            op_cnt += 1

1568
        # 1. create block 0
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
        first_block = self._get_block_by_idx(op_list1, self.partB_program, 0)

        # 2. create block 1
        second_block = self._get_block_by_idx(op_list2, self.partB_program, 1)
        # 2.1 insert send op
        self._insert_partB_communicate_op(second_block, len(op_list2))
        # 2.2 insert remain ops
        second_block = self._get_block_by_idx(op_list3, self.partB_program, 1)
        # 2.3 insert push dense grad op
        bp_op_list = get_bp_op_list(second_block)
        dense_grad_vars = self._find_dense_grad_vars(bp_op_list)
        add_send_op(self.ori_main_program, second_block, dense_grad_vars)

        # 3. insert partB recv op
        block_input_flag = "forward_joint_{}_{}@fl_ps".format(1, 2)
        grad_to_block_id = block_input_flag + ":" + str(second_block.idx)
        attrs = {
            "message_to_block_id": [grad_to_block_id],
1587
            "optimize_blocks": [second_block],  # what to do?
1588 1589 1590 1591 1592
            "endpoint": get_heter_worker_endpoint(self.role_maker),
            "fanin": len(get_previous_stage_trainers(self.role_maker)),
            "pserver_id": 1,  # TODO
            "distributed_mode": self.ps_mode,
            "rpc_exec_thread_num": int(os.getenv("CPU_NUM", 32)),
1593
            RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE,
1594
        }
1595 1596 1597 1598 1599 1600 1601
        first_block._insert_op(
            index=len(op_list1),
            type="heter_listen_and_serv",
            inputs={'X': []},
            outputs={},
            attrs=attrs,
        )
1602

1603 1604
        # logger.info('partB-first_block:{}'.format(first_block))
        # logger.info('partB-second_block:{}'.format(second_block))
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618

    def _apply_single_impl(self, main_program, startup_program, pass_ctx):
        attrs = pass_ctx._attrs
        self.role_maker = attrs['role_maker']
        self.ps_mode = attrs['ps_mode']
        self.is_part_b = attrs['is_heter_worker']  # TODO
        self.ori_main_program = main_program
        self.ori_main_block = main_program.block(0)

        party_program_map = self._split_fl_program()

        prog_a = party_program_map['a']
        _main_file = ps_log_root_dir + '6_fl_A_main_program.prototxt'
        debug_program(_main_file, prog_a)
1619 1620 1621
        self._get_partB_to_partA_grad(
            prog_a.global_block(), self.PART_A_JOINT_OP_DEVICE_FlAG
        )
1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638

        prog_b = party_program_map['b']
        _main_file = ps_log_root_dir + '6_fl_B_main_program.prototxt'
        debug_program(_main_file, prog_b)

        if not self.is_part_b:
            self.partA_program = framework.Program()
            self._get_partA_program(prog_a.global_block())
            pass_ctx._attrs['part_a_main_program'] = self.partA_program
            self._clear_op_device_flag(self.partA_program)
            check_program(self.partA_program)
        else:
            self.partB_program = framework.Program()
            self._get_partB_program(prog_b.global_block())
            pass_ctx._attrs['part_b_main_program'] = self.partB_program
            self._clear_op_device_flag(self.partB_program)
            check_program(self.partB_program)