request_handler_impl.cc 4.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <iostream>
#include <string>
#include <vector>

#include "paddle/fluid/framework/data_type.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/framework/selected_rows.h"
T
tangwei12 已提交
23
#include "paddle/fluid/operators/detail/macros.h"
24 25
#include "paddle/fluid/operators/distributed/request_handler_impl.h"
#include "paddle/fluid/operators/distributed/rpc_server.h"
T
tangwei12 已提交
26
#include "paddle/fluid/string/printf.h"
27 28 29

namespace paddle {
namespace operators {
30
namespace distributed {
31 32 33 34

bool RequestSendHandler::Handle(const std::string& varname,
                                framework::Scope* scope,
                                framework::Variable* invar,
Q
qiaolongfei 已提交
35 36
                                framework::Variable** outvar,
                                const std::string& out_var_name) {
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
  VLOG(4) << "RequestSendHandler:" << varname;

  // Async
  if (!sync_mode_) {
    try {
      executor_->RunPreparedContext((*grad_to_prepared_ctx_)[varname].get(),
                                    scope);
    } catch (std::exception& e) {
      LOG(ERROR) << "async: run sub program error " << e.what();
      return false;
    }
    return true;
  }

  // Sync
  if (varname == BATCH_BARRIER_MESSAGE) {
    VLOG(3) << "sync: recv batch barrier message";
    rpc_server_->IncreaseBatchBarrier(kRequestSend);
W
Wu Yi 已提交
55 56 57
  } else if (varname == COMPLETE_MESSAGE) {
    VLOG(3) << "sync: recv complete message";
    rpc_server_->DecreaseClientNum();
58 59 60 61 62 63 64 65 66 67 68
  } else {
    VLOG(3) << "sync: received var_name: " << varname;
    if (sync_mode_) {
      rpc_server_->WaitCond(kRequestSend);
    }

    if (invar == nullptr) {
      LOG(ERROR) << "sync: Can not find server side var: " << varname;
      PADDLE_THROW("sync: Can not find server side var");
      return false;
    }
Y
Yancey1989 已提交
69
    if (invar->IsType<framework::SelectedRows>()) {
Y
Yancey1989 已提交
70 71
      std::unique_lock<std::mutex> lock(mutex_sparse_vars_);
      sparse_vars_.push_back(invar);
Y
Yancey1989 已提交
72
    }
73 74 75 76
  }
  return true;
}

Y
Yancey1989 已提交
77 78 79 80 81 82 83 84
void RequestSendHandler::ResetSparseVarRecorder() {
  std::unique_lock<std::mutex> lock(mutex_sparse_vars_);
  for (auto* var : sparse_vars_) {
    var->GetMutable<framework::SelectedRows>()->mutable_rows()->clear();
  }
  sparse_vars_.clear();
}

85 86 87
bool RequestGetHandler::Handle(const std::string& varname,
                               framework::Scope* scope,
                               framework::Variable* invar,
Q
qiaolongfei 已提交
88 89
                               framework::Variable** outvar,
                               const std::string& out_var_name) {
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
  VLOG(4) << "RequestGetHandler:" << varname;

  if (varname != FETCH_BARRIER_MESSAGE) {
    if (sync_mode_) {
      rpc_server_->WaitCond(kRequestGet);
    }
    *outvar = scope_->FindVar(varname);
    return true;
  }

  // FETCH_BARRIER_MESSAGE
  if (sync_mode_) {
    VLOG(3) << "sync: recv fetch barrier message";
    rpc_server_->IncreaseBatchBarrier(kRequestGet);
  }

  return true;
}

bool RequestPrefetchHandler::Handle(const std::string& varname,
                                    framework::Scope* scope,
                                    framework::Variable* invar,
Q
qiaolongfei 已提交
112 113
                                    framework::Variable** outvar,
                                    const std::string& out_var_name) {
114 115
  VLOG(4) << "RequestPrefetchHandler " << varname;

Q
qiaolongfei 已提交
116
  auto var_desc = program_->Block(0).FindVar(out_var_name);
117
  InitializeVariable(*outvar, var_desc->GetType());
118 119
  executor_->RunPreparedContext(
      (*prefetch_var_name_to_prepared_ctx_)[varname].get(), scope);
120 121 122 123

  return true;
}

T
tangwei12 已提交
124 125 126 127
bool RequestCheckpointHandler::Handle(const std::string& varname,
                                      framework::Scope* scope,
                                      framework::Variable* invar,
                                      framework::Variable** outvar,
T
bug fix  
tangwei12 已提交
128
                                      const std::string& out_var_name) {
129 130 131
  PADDLE_ENFORCE(
      checkpoint_notify_id != -1,
      "when checkpoint_notify_id = -1, there should be no RPC invoke.");
T
tangwei12 已提交
132

T
tangwei12 已提交
133
  auto* lt_var = scope->FindVar(LOOKUP_TABLE_PATH)->GetMutable<std::string>();
T
tangwei12 已提交
134 135
  lt_var->clear();
  lt_var->append(out_var_name);
T
tangwei12 已提交
136
  VLOG(4) << "RequestCheckpointHandler update var lookup_table_path to: "
137
          << out_var_name;
T
bug fix  
tangwei12 已提交
138
  executor_->RunPreparedContext(checkpoint_prepared_ctx_.get(), scope);
T
bug fix  
tangwei12 已提交
139 140
  return true;
}
T
tangwei12 已提交
141

142
}  // namespace distributed
143 144
}  // namespace operators
}  // namespace paddle