multi_devices_graph_builder.cc 22.3 KB
Newer Older
Y
Yu Yang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
C
chengduoZH 已提交
14
#include <algorithm>
Y
Yancey1989 已提交
15
#include <fstream>
C
chengduoZH 已提交
16
#include <string>
C
chengduoZH 已提交
17
#include <utility>
C
chengduoZH 已提交
18 19
#include <vector>

20
#include "paddle/fluid/framework/details/all_reduce_op_handle.h"
C
chengduoZH 已提交
21
#include "paddle/fluid/framework/details/broadcast_op_handle.h"
Y
Yu Yang 已提交
22
#include "paddle/fluid/framework/details/computation_op_handle.h"
23
#include "paddle/fluid/framework/details/data_balance_op_handle.h"
C
chengduoZH 已提交
24
#include "paddle/fluid/framework/details/multi_devices_graph_builder.h"
C
chengduoZH 已提交
25
#include "paddle/fluid/framework/details/reduce_op_handle.h"
Y
Yancey1989 已提交
26
#include "paddle/fluid/framework/details/rpc_op_handle.h"
Y
Yu Yang 已提交
27
#include "paddle/fluid/framework/details/scale_loss_grad_op_handle.h"
Y
Fix bug  
yuyang18 已提交
28
#include "paddle/fluid/framework/op_info.h"
Y
Yu Yang 已提交
29
#include "paddle/fluid/framework/scope.h"
Y
Yu Yang 已提交
30

Y
Yu Yang 已提交
31 32 33
namespace paddle {
namespace framework {
namespace details {
Y
Yu Yang 已提交
34 35

#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
36 37 38 39
MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder(
    const std::vector<platform::Place> &places,
    const std::string &loss_var_name,
    const std::unordered_set<std::string> &params,
C
chengduoZH 已提交
40
    const std::vector<Scope *> &local_scopes,
Y
yuyang18 已提交
41
    platform::NCCLContextMap *nccl_ctxs, const BuildStrategy &strategy)
Y
Yu Yang 已提交
42 43 44
    : loss_var_name_(loss_var_name),
      places_(places),
      local_scopes_(local_scopes),
C
chengduoZH 已提交
45
      nccl_ctxs_(nccl_ctxs),
Y
yuyang18 已提交
46
      strategy_(strategy) {
Y
Yu Yang 已提交
47 48 49 50 51
#else
MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder(
    const std::vector<platform::Place> &places,
    const std::string &loss_var_name,
    const std::unordered_set<std::string> &params,
Y
yuyang18 已提交
52
    const std::vector<Scope *> &local_scopes, const BuildStrategy &strategy)
Y
Yu Yang 已提交
53 54
    : loss_var_name_(loss_var_name),
      places_(places),
C
chengduoZH 已提交
55
      local_scopes_(local_scopes),
Y
yuyang18 已提交
56
      strategy_(strategy) {
Y
Yu Yang 已提交
57
#endif
Y
Yu Yang 已提交
58 59 60
  for (auto &p : params) {
    grad_names_.insert(GradVarName(p));
  }
Y
Yancey1989 已提交
61
  balance_vars_.resize(places_.size(), 0);
Y
yuyang18 已提交
62 63 64 65 66
  if (strategy_.enable_data_balance_ && places_.size() == 1) {
    LOG(WARNING) << "It is no need to enable data balance when there is only "
                    "one place. enable_data_balance is set to False.";
    strategy_.enable_data_balance_ = false;
  }
Y
Yu Yang 已提交
67 68
}

Y
Yu Yang 已提交
69 70
void MultiDevSSAGraphBuilder::CreateOpHandleIOs(SSAGraph *result,
                                                const OpDesc &op,
Y
Yu Yang 已提交
71 72
                                                size_t place_id) const {
  auto p = places_[place_id];
T
wip  
typhoonzero 已提交
73
  auto *op_handle = result->ops_.back().get();
X
Xin Pan 已提交
74 75
  op_handle->SetDeviceContext(p,
                              platform::DeviceContextPool::Instance().Get(p));
T
wip  
typhoonzero 已提交
76

Y
Yu Yang 已提交
77 78 79
  for (auto &each_var_name : op.InputArgumentNames()) {
    VarHandle *var =
        CreateOrGetLatestVarHandle(result, each_var_name, p, place_id);
T
wip  
typhoonzero 已提交
80 81 82
    op_handle->AddInput(var);
  }

Y
Yu Yang 已提交
83 84
  for (auto &each_var_name : op.OutputArgumentNames()) {
    CreateOpOutput(result, op_handle, each_var_name, p, place_id);
T
wip  
typhoonzero 已提交
85 86
  }
}
Y
fix pe  
Yancey1989 已提交
87 88 89 90

std::vector<std::string> MultiDevSSAGraphBuilder::FindDistTrainSendVars(
    const ProgramDesc &program) const {
  std::vector<std::string> send_vars;
Y
Yancey1989 已提交
91 92
  // since parameters are all in block 0,
  // it's enough to only scan send ops in block 0
Y
fix pe  
Yancey1989 已提交
93
  for (auto *op : program.Block(0).AllOps()) {
Y
Yancey1989 已提交
94 95
    // TODO(Yancey1989): use a graceful method to find send op,
    // instead of the the hard code string
96
    if (op->Type() == "send") {
Y
fix pe  
Yancey1989 已提交
97 98 99 100 101 102 103 104 105 106 107 108 109
      auto op_vars = op->InputArgumentNames();
      send_vars.reserve(send_vars.size() +
                        std::distance(op_vars.begin(), op_vars.end()));
      send_vars.insert(send_vars.end(), op_vars.begin(), op_vars.end());
    }
  }
  return send_vars;
}

std::vector<std::string> MultiDevSSAGraphBuilder::FindDistTrainRecvVars(
    const ProgramDesc &program) const {
  std::vector<std::string> recv_vars;
  for (auto *op : program.Block(0).AllOps()) {
Y
Yancey1989 已提交
110 111 112
    // TODO(Yancey1989): use a graceful method to find recv op,
    // instead of the hard code string
    if (op->Type() == "recv") {
Y
fix pe  
Yancey1989 已提交
113 114 115 116 117 118 119 120 121 122 123 124 125
      auto op_vars = op->OutputArgumentNames();
      recv_vars.reserve(recv_vars.size() +
                        std::distance(op_vars.begin(), op_vars.end()));
      recv_vars.insert(recv_vars.end(), op_vars.begin(), op_vars.end());
    }
  }
  return recv_vars;
}

bool MultiDevSSAGraphBuilder::IsDistTrainOp(
    const OpDesc &op, const std::vector<std::string> &send_vars,
    const std::vector<std::string> &recv_vars) const {
  if (send_vars.size() == 0 || recv_vars.size() == 0) {
T
typhoonzero 已提交
126 127 128
    return false;
  }

Y
Yu Yang 已提交
129 130 131 132
  /**
   * Check any of opvars contains `.block` and in sendvars
   */
  auto checker = [](const std::vector<std::string> &opvars,
Y
fix pe  
Yancey1989 已提交
133
                    const std::vector<std::string> &rpc_vars) -> bool {
T
typhoonzero 已提交
134
    for (auto &var : opvars) {
Y
Yancey1989 已提交
135 136 137
      // a variable name with the suffix `.block` means it's a splited
      // variable by (DistributeTranspiler)
      // [python/paddle/fluid/transpiler/distribute_transpiler.py]
T
typhoonzero 已提交
138
      if (var.find(".block") != std::string::npos &&
Y
fix pe  
Yancey1989 已提交
139
          std::find(rpc_vars.begin(), rpc_vars.end(), var) != rpc_vars.end()) {
Y
Yu Yang 已提交
140
        return true;
T
typhoonzero 已提交
141 142
      }
    }
Y
Yu Yang 已提交
143
    return false;
T
typhoonzero 已提交
144 145
  };

Y
Yancey1989 已提交
146 147
  return checker(op.OutputArgumentNames(), send_vars) ||
         checker(op.InputArgumentNames(), recv_vars);
T
typhoonzero 已提交
148 149
}

Y
Yancey1989 已提交
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
size_t MultiDevSSAGraphBuilder::GetAppropriateDeviceID(
    const std::vector<std::string> &var_names) const {
  int64_t numel_sum = 0;
  for (auto var_name : var_names) {
    auto var_desc = all_vars_.at(var_name);
    PADDLE_ENFORCE_NOT_NULL(var_desc);
    auto dim = framework::make_ddim(var_desc->GetShape());
    int64_t numel = framework::product(dim);
    PADDLE_ENFORCE_GT(numel, 0);
    numel_sum += numel;
  }

  auto smallest =
      std::min_element(std::begin(balance_vars_), std::end(balance_vars_));
  size_t dev_id =
      static_cast<size_t>(std::distance(std::begin(balance_vars_), smallest));
  balance_vars_[dev_id] += numel_sum;
  return dev_id;
}

Y
Yu Yang 已提交
170 171
std::unique_ptr<SSAGraph> MultiDevSSAGraphBuilder::Build(
    const ProgramDesc &program) const {
C
fix ci  
chengduoZH 已提交
172
  for (auto *var : program.Block(0).AllVars()) {
Y
Yancey1989 已提交
173
    all_vars_.emplace(var->Name(), var);
C
fix ci  
chengduoZH 已提交
174
  }
C
chengduoZH 已提交
175

Y
Yu Yang 已提交
176
  auto graph = new SSAGraph();
Y
Yu Yang 已提交
177
  SSAGraph &result = *graph;
C
chengduoZH 已提交
178
  std::unordered_set<std::string> og_has_been_broadcast;
Y
Yu Yang 已提交
179 180 181 182 183

  // We cannot invoke resize. It is a bug of GCC 4.8
  result.vars_ = std::vector<
      std::unordered_map<std::string, std::vector<std::unique_ptr<VarHandle>>>>(
      places_.size());
Y
Yu Yang 已提交
184

Y
fix pe  
Yancey1989 已提交
185 186 187 188
  // find send/recv vars so that we can place the distributed training
  // realted op in the place 0
  auto send_vars = FindDistTrainSendVars(program);
  auto recv_vars = FindDistTrainRecvVars(program);
T
typhoonzero 已提交
189

C
chengduoZH 已提交
190 191 192
  std::vector<std::unordered_set<std::string>> bcast_var_name_set;
  bcast_var_name_set.resize(places_.size());

C
chengduoZH 已提交
193
  size_t cur_device_id = 0;
Y
Yu Yang 已提交
194
  bool is_forwarding = true;
195

Y
Yu Yang 已提交
196
  for (auto *op : program.Block(0).AllOps()) {
Y
Yancey1989 已提交
197 198 199
    if (boost::get<int>(
            op->GetAttr(OpProtoAndCheckerMaker::OpRoleAttrName())) ==
        static_cast<int>(OpRole::kRPC)) {
Y
Yancey1989 已提交
200
      CreateRPCOp(&result, *op);
Y
fix pe  
Yancey1989 已提交
201
    } else if (IsDistTrainOp(*op, send_vars, recv_vars)) {
Y
Yancey1989 已提交
202
      CreateDistTrainOp(&result, *op);
Y
Yu Yang 已提交
203
    } else if (IsScaleLossOp(*op)) {
Y
Yu Yang 已提交
204
      // user can customize loss@grad if not use_default_grad_scale_
Y
yuyang18 已提交
205 206
      if (strategy_.gradient_scale_ !=
          BuildStrategy::GradientScaleStrategy::kCustomized) {
Y
Yu Yang 已提交
207 208
        CreateScaleLossGradOp(&result);
      }
209 210 211 212
      // This assumes the backward generating code will ensure IsScaleLossOp
      // is true only for the op that scale the final scalar loss.
      // It also assumes backward op will always follow the forward op in
      // the block.
Y
Yu Yang 已提交
213
      is_forwarding = false;
Y
Yu Yang 已提交
214
    } else {
215
      int op_dev_id = GetOpDeviceID(*op);
C
chengduo 已提交
216
      if (op_dev_id != -1) {  // This op only runs on one specific device.
C
chengduoZH 已提交
217 218
        CreateComputationalOp(&result, *op, op_dev_id);
        for (auto &var_name : op->OutputArgumentNames()) {
219
          var_name_on_devices_.emplace(var_name, op_dev_id);
C
chengduoZH 已提交
220
        }
C
chengduo 已提交
221 222 223
      } else {
        // This op runs on all devices, and its output may have parameter's
        // gradients.
F
fengjiayi 已提交
224
        if (op->Type() == "read" && strategy_.enable_data_balance_) {
F
fengjiayi 已提交
225 226
          op->SetAttr("throw_eof_exp", false);
          CreateComputationalOps(&result, *op, places_.size());
227 228
          const auto &data_var_names = op->Output("Out");
          InsertDataBalanceOp(&result, data_var_names);
F
fengjiayi 已提交
229 230
        } else {
          CreateComputationalOps(&result, *op, places_.size());
231 232
        }

C
chengduo 已提交
233 234 235 236 237 238 239 240 241 242
        if (!is_forwarding && places_.size() > 1) {
          // Currently, we assume that once gradient is generated, it can be
          // broadcast, and each gradient is only broadcast once.
          if (static_cast<bool>(boost::get<int>(op->GetAttr(
                                    OpProtoAndCheckerMaker::OpRoleAttrName())) &
                                static_cast<int>(OpRole::kBackward))) {
            try {
              auto backward_vars =
                  boost::get<std::vector<std::string>>(op->GetNullableAttr(
                      OpProtoAndCheckerMaker::OpRoleVarAttrName()));
Y
yuyang18 已提交
243

C
chengduo 已提交
244
              PADDLE_ENFORCE_EQ(backward_vars.size() % 2, 0);
Y
yuyang18 已提交
245

C
chengduo 已提交
246 247 248 249
              for (size_t i = 0; i < backward_vars.size(); i += 2) {
                auto &p_name = backward_vars[i];
                auto &g_name = backward_vars[i + 1];
                VLOG(10) << "Bcast " << g_name << " for parameter " << p_name;
Y
yuyang18 已提交
250

C
chengduo 已提交
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
                switch (strategy_.reduce_) {
                  case BuildStrategy::ReduceStrategy::kReduce:
                    cur_device_id = GetAppropriateDeviceID({g_name});
                    CreateReduceOp(&result, g_name, cur_device_id);
                    var_name_on_devices_.emplace(g_name, cur_device_id);
                    bcast_var_name_set[cur_device_id].emplace(p_name);
                    break;
                  case BuildStrategy::ReduceStrategy::kAllReduce:
                    if (IsSparseGradient(g_name)) {
                      CreateReduceOp(&result, g_name, 0);
                      CreateBroadcastOp(&result, g_name, 0);
                    } else {
                      InsertAllReduceOp(&result, g_name);
                    }
                    break;
                  default:
                    LOG(FATAL) << "Unknown reduce strategy ";
                    break;
                }
Y
yuyang18 已提交
270
              }
C
chengduo 已提交
271
            } catch (boost::bad_get e) {
C
chengduoZH 已提交
272
            }
Y
Yu Yang 已提交
273 274 275 276 277 278
          }
        }
      }
    }
  }

279 280 281 282 283 284 285 286 287 288 289 290 291
  bool use_gpu = false;
#ifdef PADDLE_WITH_CUDA
  use_gpu = nccl_ctxs_ != nullptr;
#endif

  if (use_gpu ||
      strategy_.reduce_ == BuildStrategy::ReduceStrategy::kAllReduce) {
    // Insert BCast Ops
    for (size_t dev_id = 0; dev_id < bcast_var_name_set.size(); ++dev_id) {
      auto &to_bcast_set = bcast_var_name_set[dev_id];
      for (auto &bcast_name : to_bcast_set) {
        CreateBroadcastOp(&result, bcast_name, dev_id);
      }
C
chengduoZH 已提交
292 293
    }
  }
294

Y
Yu Yang 已提交
295 296
  /*
    Dependency graph has been constructed. However, there are still data
297
    hazards need to be handled.
Y
Yu Yang 已提交
298 299
   */
  PolishGraphToSupportDataHazards(&result);
Y
Yu Yang 已提交
300

Y
Yu Yang 已提交
301 302 303 304 305
  /*
   * Only variables should be the leaves of graph.
   */
  AddOutputToLeafOps(&result);

Y
Yu Yang 已提交
306
  return std::unique_ptr<SSAGraph>(graph);
Y
Yu Yang 已提交
307 308
}

Y
Yancey1989 已提交
309 310 311
bool MultiDevSSAGraphBuilder::IsSparseGradient(const std::string &og) const {
  PADDLE_ENFORCE(all_vars_.count(og) != 0);
  if (all_vars_.at(og)->GetType() == proto::VarType::SELECTED_ROWS) {
C
fix ci  
chengduoZH 已提交
312 313 314
    return true;
  }
  return false;
315 316
}

317 318 319 320 321 322 323 324 325 326 327 328 329
void MultiDevSSAGraphBuilder::SetCommunicationContext(
    OpHandleBase *op_handle, const platform::Place &p) const {
#ifdef PADDLE_WITH_CUDA
  if (nccl_ctxs_ == nullptr) {
    op_handle->SetDeviceContext(p,
                                platform::DeviceContextPool::Instance().Get(p));
  }
#else
  op_handle->SetDeviceContext(p,
                              platform::DeviceContextPool::Instance().Get(p));
#endif
}

C
chengduoZH 已提交
330 331
void MultiDevSSAGraphBuilder::CreateBroadcastOp(SSAGraph *result,
                                                const std::string &p_name,
C
chengduoZH 已提交
332
                                                size_t src_dev_id) const {
C
chengduoZH 已提交
333 334 335 336 337 338 339
#ifdef PADDLE_WITH_CUDA
  auto *op_handle = new BroadcastOpHandle(local_scopes_, places_, nccl_ctxs_);
#else
  auto *op_handle = new BroadcastOpHandle(local_scopes_, places_);
#endif

  result->ops_.emplace_back(op_handle);
C
chengduoZH 已提交
340
  auto *in = result->vars_.at(src_dev_id).at(p_name).back().get();
C
chengduoZH 已提交
341 342 343 344
  op_handle->AddInput(in);

  for (size_t i = 0; i < places_.size(); ++i) {
    auto &p = places_[i];
C
chengduoZH 已提交
345 346
    SetCommunicationContext(op_handle, p);
    auto &vars = result->vars_.at(i).at(p_name);
C
chengduoZH 已提交
347 348 349 350 351 352 353 354 355 356 357 358 359 360
    auto *out_var = new VarHandle(vars.size(), i, p_name, p);
    vars.emplace_back(out_var);
    op_handle->AddOutput(out_var);
  }
}

void MultiDevSSAGraphBuilder::CreateComputationalOp(SSAGraph *result,
                                                    const OpDesc &op,
                                                    int dev_id) const {
  result->ops_.emplace_back(
      new ComputationOpHandle(op, local_scopes_[dev_id], places_[dev_id]));
  CreateOpHandleIOs(result, op, dev_id);
}

C
chengduoZH 已提交
361 362
void MultiDevSSAGraphBuilder::InsertAllReduceOp(SSAGraph *result,
                                                const std::string &og) const {
Y
Yu Yang 已提交
363 364
#ifdef PADDLE_WITH_CUDA
  result->ops_.emplace_back(
365
      new AllReduceOpHandle(local_scopes_, places_, nccl_ctxs_));
C
chengduoZH 已提交
366
#else
367
  result->ops_.emplace_back(new AllReduceOpHandle(local_scopes_, places_));
C
chengduoZH 已提交
368
#endif
Y
Yu Yang 已提交
369 370 371 372
  auto *op_handle = result->ops_.back().get();

  for (size_t i = 0; i < places_.size(); ++i) {
    auto &p = places_[i];
C
chengduoZH 已提交
373
    SetCommunicationContext(op_handle, p);
Y
Yu Yang 已提交
374
    auto &vars = result->vars_[i][og];
Y
Yu Yang 已提交
375 376
    PADDLE_ENFORCE(!vars.empty());
    auto &prev_grad = vars.back();
Y
Yu Yang 已提交
377 378
    op_handle->AddInput(prev_grad.get());

379
    auto var = new VarHandle(vars.size(), i, og, p);
Y
Yu Yang 已提交
380 381 382 383 384
    vars.emplace_back(var);
    op_handle->AddOutput(var);
  }
}

385 386
void MultiDevSSAGraphBuilder::InsertDataBalanceOp(
    SSAGraph *result, const std::vector<std::string> &datas) const {
F
fengjiayi 已提交
387 388 389 390
#ifdef PADDLE_WITH_CUDA
  result->ops_.emplace_back(
      new DataBalanceOpHandle(local_scopes_, places_, nccl_ctxs_));
#else
391
  result->ops_.emplace_back(new DataBalanceOpHandle(local_scopes_, places_));
F
fengjiayi 已提交
392
#endif
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
  auto *op_handle = result->ops_.back().get();
  for (size_t i = 0; i < places_.size(); ++i) {
    auto &p = places_[i];
    SetCommunicationContext(op_handle, p);
    for (const std::string &d_name : datas) {
      auto &vars = result->vars_[i][d_name];
      PADDLE_ENFORCE(!vars.empty());
      op_handle->AddInput(vars.back().get());
      auto var = new VarHandle(vars.size(), i, d_name, p);
      vars.emplace_back(var);
      op_handle->AddOutput(var);
    }
  }
}

Y
Yu Yang 已提交
408 409 410 411 412 413 414 415 416 417 418 419
bool MultiDevSSAGraphBuilder::IsParameterGradientOnce(
    const std::string &og,
    std::unordered_set<std::string> *og_has_been_broadcast) const {
  bool is_pg_once =
      grad_names_.count(og) != 0 && og_has_been_broadcast->count(og) == 0;
  if (is_pg_once) {
    // Insert NCCL AllReduce Op
    og_has_been_broadcast->insert(og);
  }
  return is_pg_once;
}

420
int MultiDevSSAGraphBuilder::GetOpDeviceID(const OpDesc &op) const {
Y
yuyang18 已提交
421
  if (strategy_.reduce_ != BuildStrategy::ReduceStrategy::kReduce) {
C
chengduoZH 已提交
422 423
    return -1;
  }
424 425 426 427
  int op_role = boost::get<int>(
      op.GetAttr(framework::OpProtoAndCheckerMaker::OpRoleAttrName()));
  if (op_role != static_cast<int>(framework::OpRole::kOptimize)) {
    return -1;
C
chengduoZH 已提交
428
  }
429 430 431 432 433 434 435 436
  auto param_grad = boost::get<std::vector<std::string>>(
      op.GetAttr(OpProtoAndCheckerMaker::OpRoleVarAttrName()));

  PADDLE_ENFORCE_EQ(param_grad.size(), 2U);
  int dev_id = GetVarDeviceID(param_grad[1]);
  PADDLE_ENFORCE_NE(dev_id, -1, "dev_id should not be -1.[%s, %s]", op.Type(),
                    param_grad[0]);
  return dev_id;
437 438 439 440 441
}

int MultiDevSSAGraphBuilder::GetVarDeviceID(const std::string &varname) const {
  auto got = var_name_on_devices_.find(varname);
  return got == var_name_on_devices_.end() ? -1 : got->second;
C
chengduoZH 已提交
442 443
}

Y
Yu Yang 已提交
444 445 446 447
void MultiDevSSAGraphBuilder::CreateScaleLossGradOp(SSAGraph *result) const {
  for (size_t i = 0; i < places_.size(); ++i) {
// Insert ScaleCost OpHandle
#ifdef PADDLE_WITH_CUDA
C
chengduoZH 已提交
448 449 450
    auto *communication_dev_ctx =
        nccl_ctxs_ ? nccl_ctxs_->DevCtx(places_[i])
                   : platform::DeviceContextPool::Instance().Get(places_[i]);
Y
Yu Yang 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
#else
    auto *communication_dev_ctx =
        platform::DeviceContextPool::Instance().Get(platform::CPUPlace());
#endif

    auto *op_handle =
        new ScaleLossGradOpHandle(local_scopes_.size(), local_scopes_[i],
                                  places_[i], communication_dev_ctx);
    result->ops_.emplace_back(op_handle);

    // FIXME: Currently ScaleLossGradOp only use device_count as scale
    // factor. So it does not depend on any other operators.
    // VarHandle *loss = GetVarHandle(loss_var_name, place);
    // loss->pending_ops_.emplace_back(op_handle);
    // op_handle->inputs_.emplace_back(loss);

    CreateOpOutput(result, op_handle, GradVarName(loss_var_name_), places_[i],
                   i);
  }
}

void MultiDevSSAGraphBuilder::CreateComputationalOps(SSAGraph *result,
T
typhoonzero 已提交
473 474 475
                                                     const OpDesc &op,
                                                     size_t num_places) const {
  for (size_t scope_idx = 0; scope_idx < num_places; ++scope_idx) {
Y
Yu Yang 已提交
476 477 478
    auto p = places_[scope_idx];
    auto s = local_scopes_[scope_idx];
    result->ops_.emplace_back(new ComputationOpHandle(op, s, p));
Y
Yu Yang 已提交
479
    CreateOpHandleIOs(result, op, scope_idx);
Y
Yu Yang 已提交
480 481 482
  }
}

C
chengduoZH 已提交
483 484 485
VarHandle *MultiDevSSAGraphBuilder::CreateReduceOp(SSAGraph *result,
                                                   const std::string &og,
                                                   int dst_dev_id) const {
C
chengduoZH 已提交
486 487 488 489 490 491 492 493 494 495
#ifdef PADDLE_WITH_CUDA
  result->ops_.emplace_back(
      new ReduceOpHandle(local_scopes_, places_, nccl_ctxs_));
#else
  result->ops_.emplace_back(new ReduceOpHandle(local_scopes_, places_));
#endif
  auto *op_handle = result->ops_.back().get();

  for (size_t i = 0; i < places_.size(); ++i) {
    auto &p = places_[i];
C
chengduoZH 已提交
496 497
    SetCommunicationContext(op_handle, p);
    auto &vars = result->vars_[i][og];
C
chengduoZH 已提交
498 499 500 501 502
    PADDLE_ENFORCE(!vars.empty());
    auto &prev_grad = vars.back();
    op_handle->AddInput(prev_grad.get());
  }
  auto &vars = result->vars_[dst_dev_id][og];
503
  auto var = new VarHandle(vars.size(), dst_dev_id, og, places_[dst_dev_id]);
C
chengduoZH 已提交
504 505 506 507 508
  vars.emplace_back(var);
  op_handle->AddOutput(var);
  return var;
}

509 510
// Find the first occurence of `prev_op_name` and make current `op` depend
// on it.
Y
fix pe  
Yancey1989 已提交
511 512
void MultiDevSSAGraphBuilder::ConnectOp(SSAGraph *result, OpHandleBase *op,
                                        const std::string &prev_op_name) const {
Y
Yancey1989 已提交
513
  for (auto &prev_op : result->ops_) {
Y
fix pe  
Yancey1989 已提交
514
    if (prev_op->Name() == prev_op_name) {
Y
Yancey1989 已提交
515 516 517
      auto *dep_var = new DummyVarHandle();
      prev_op->AddOutput(dep_var);
      result->dep_vars_.emplace(dep_var);
Y
fix pe  
Yancey1989 已提交
518
      op->AddInput(dep_var);
Y
Yancey1989 已提交
519 520 521 522
    }
  }
}

Y
Yancey1989 已提交
523
void MultiDevSSAGraphBuilder::CreateDistTrainOp(SSAGraph *result,
Y
Yancey1989 已提交
524 525
                                                const OpDesc &op) const {
  int op_dev_id = -1;
Y
yi.wu 已提交
526
  if (op.Type() == "split_byref" || op.Type() == "split_selected_rows") {
Y
Yancey1989 已提交
527 528 529 530 531 532 533 534 535 536 537 538
    op_dev_id = GetVarDeviceID(op.InputArgumentNames()[0]);
    if (strategy_.reduce_ == BuildStrategy::ReduceStrategy::kAllReduce) {
      op_dev_id = GetAppropriateDeviceID(op.InputArgumentNames());
      for (auto &varname : op.InputArgumentNames()) {
        var_name_on_devices_.emplace(varname, op_dev_id);
      }
    }
    for (auto &varname : op.OutputArgumentNames()) {
      var_name_on_devices_.emplace(varname, op_dev_id);
    }
  } else if (op.Type() == "concat") {
    op_dev_id = GetVarDeviceID(op.InputArgumentNames()[0]);
Y
yi.wu 已提交
539 540 541
    for (auto &varname : op.OutputArgumentNames()) {
      var_name_on_devices_.emplace(varname, op_dev_id);
    }
Y
Yancey1989 已提交
542 543 544 545 546 547 548 549 550 551
  } else {
    PADDLE_ENFORCE(
        "the distribute training related op should be in [split_byref, "
        "concat].");
  }

  PADDLE_ENFORCE(op_dev_id != -1,
                 "can not find right place for distributed op: %s", op.Type());

  CreateComputationalOp(result, op, op_dev_id);
Y
Yancey1989 已提交
552 553 554 555 556
  if (op.Type() == "concat") {
    ConnectOp(result, result->ops_.back().get(), "fetch_barrier");
  }
}

557
// Create RPC related op handles that connects its in ops and out ops.
Y
Yancey1989 已提交
558 559 560 561 562 563 564
void MultiDevSSAGraphBuilder::CreateRPCOp(SSAGraph *result,
                                          const OpDesc &op) const {
  int op_dev_id = -1;
  if (op.Type() == "send") {
    op_dev_id = GetVarDeviceID(op.InputArgumentNames()[0]);
    // the variable name which contains .block means it was splited by
    // split_byref op
565 566
    // so that we can balance the variable blocks to all the pserver
    // instances.
Y
Yancey1989 已提交
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588
    if (strategy_.reduce_ == BuildStrategy::ReduceStrategy::kAllReduce &&
        op.InputArgumentNames()[0].find(".block") == std::string::npos) {
      op_dev_id = GetAppropriateDeviceID(op.InputArgumentNames());
      for (auto &varname : op.InputArgumentNames()) {
        var_name_on_devices_.emplace(varname, op_dev_id);
      }
    }
  } else if (op.Type() == "recv") {
    op_dev_id = GetAppropriateDeviceID(op.OutputArgumentNames());
    for (auto &varname : op.OutputArgumentNames()) {
      var_name_on_devices_.emplace(varname, op_dev_id);
    }
  } else {
    // send_barrier and fetch_barrier op can be scheduled on device 0
    op_dev_id = 0;
  }

  PADDLE_ENFORCE(op_dev_id != -1, "can not find the right place for rpc op: %s",
                 op.Type());

  result->ops_.emplace_back(new RPCOpHandle(op, local_scopes_[op_dev_id],
                                            op.Type(), places_[op_dev_id]));
Y
fix pe  
Yancey1989 已提交
589

Y
Yancey1989 已提交
590
  if (op.Type() == "send_barrier") {
591
    ConnectOp(result, result->ops_.back().get(), "send");
Y
Yancey1989 已提交
592
  } else if (op.Type() == "recv") {
Y
fix pe  
Yancey1989 已提交
593
    ConnectOp(result, result->ops_.back().get(), "send_barrier");
Y
Yancey1989 已提交
594
  } else if (op.Type() == "fetch_barrier") {
Y
fix pe  
Yancey1989 已提交
595
    ConnectOp(result, result->ops_.back().get(), "recv");
596
  } else if (op.Type() == "send") {
Y
Yancey1989 已提交
597 598 599
    // do nothing
  } else {
    PADDLE_THROW(
Y
Yancey1989 已提交
600
        "rpc op should be in ["
601
        "send, send_barrier. recv, fetch_barrier]");
Y
Yancey1989 已提交
602 603
  }

Y
Yancey1989 已提交
604
  CreateOpHandleIOs(result, op, op_dev_id);
Y
Yu Yang 已提交
605 606 607
}

bool MultiDevSSAGraphBuilder::IsScaleLossOp(const OpDesc &op) const {
Y
yuyang18 已提交
608 609
  return boost::get<int>(
             op.GetAttr(OpProtoAndCheckerMaker::OpRoleAttrName())) ==
Y
Fix bug  
yuyang18 已提交
610 611 612
             (static_cast<int>(OpRole::kBackward) |
              static_cast<int>(OpRole::kLoss)) &&
         !loss_var_name_.empty();  // If loss_var is empty. This is test mode
Y
Yu Yang 已提交
613
}
Y
Yu Yang 已提交
614 615 616
}  // namespace details
}  // namespace framework
}  // namespace paddle