parallel_executor.h 3.2 KB
Newer Older
Y
Yang Yang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#pragma once

Q
Qiao Longfei 已提交
17
#include <memory>
X
Xin Pan 已提交
18
#include <string>
S
sneaxiy 已提交
19
#include <unordered_map>
Y
Yang Yang 已提交
20
#include <unordered_set>
Y
Yan Xu 已提交
21
#include <utility>
X
Xin Pan 已提交
22
#include <vector>
23 24

#include "paddle/fluid/framework/details/build_strategy.h"
Y
yuyang18 已提交
25
#include "paddle/fluid/framework/details/execution_strategy.h"
26
#include "paddle/fluid/framework/details/op_handle_base.h"
Y
Yang Yang 已提交
27
#include "paddle/fluid/framework/executor.h"
28
#include "paddle/fluid/framework/feed_fetch_type.h"
Y
Yang Yang 已提交
29 30 31 32 33
#include "paddle/fluid/framework/op_info.h"
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/framework/tensor.h"
#include "paddle/fluid/platform/device_context.h"
34

35
#if defined(PADDLE_WITH_NCCL)
Y
Yancey1989 已提交
36 37 38
#include "paddle/fluid/platform/nccl_helper.h"
#endif

Y
Yang Yang 已提交
39 40 41
namespace paddle {
namespace framework {

Y
Yu Yang 已提交
42
class ParallelExecutorPrivate;
Y
Use mtx  
Yu Yang 已提交
43

Y
yuyang18 已提交
44
using details::BuildStrategy;
Y
yuyang18 已提交
45 46
using details::ExecutionStrategy;

Y
Yang Yang 已提交
47
class ParallelExecutor {
Y
Yu Yang 已提交
48 49
  DISABLE_COPY_AND_ASSIGN(ParallelExecutor);

Y
Yu Yang 已提交
50
 public:
Y
yuyang18 已提交
51
  explicit ParallelExecutor(const std::vector<platform::Place> &places,
Y
Yan Xu 已提交
52
                            const std::vector<std::string> &bcast_vars,
Y
yuyang18 已提交
53 54
                            const std::string &loss_var_name, Scope *scope,
                            const std::vector<Scope *> &local_scopes,
Y
yuyang18 已提交
55
                            const ExecutionStrategy &exec_strategy,
X
Xin Pan 已提交
56
                            const BuildStrategy &build_strategy,
Q
Qiao Longfei 已提交
57
                            ir::Graph *graph);
Y
Yu Yang 已提交
58

59 60
  ~ParallelExecutor();

61 62
  size_t DeviceCount() const;

Y
yuyang18 已提交
63
  std::vector<Scope *> &GetLocalScopes();
64

65 66 67 68 69
  void DropLocalExeScopes();

  // This API is used to check whether DropLocalExeScopes work.
  bool NeedCreateLocalExeScope();

Y
Yu Yang 已提交
70 71 72 73 74
  /**
   * Feed tensors to local scopes. The size of tensors should be equal to the
   * size of local scopes.
   */
  void FeedTensorsIntoLocalScopes(
Y
yuyang18 已提交
75
      const std::vector<std::unordered_map<std::string, LoDTensor>> &tensors);
Y
Yu Yang 已提交
76 77

  void FeedAndSplitTensorIntoLocalScopes(
Y
yuyang18 已提交
78
      const std::unordered_map<std::string, LoDTensor> &tensors);
Y
Yu Yang 已提交
79

80
  FeedFetchList Run(const std::vector<std::string> &fetch_tensors);
Y
Yang Yang 已提交
81

X
Xin Pan 已提交
82
 private:
Y
Yan Xu 已提交
83 84 85 86
  // broadcast the parameters from the 0th device.
  // trainer_id the trainer index in nccl distributed training.
  void BCastParamsToDevices(const std::vector<std::string> &vars,
                            int trainer_id = 0) const;
X
Xin Pan 已提交
87 88 89
  bool EnableParallelGraphExecution(const ir::Graph &graph,
                                    const ExecutionStrategy &exec_strategy,
                                    const BuildStrategy &build_strategy) const;
T
typhoonzero 已提交
90

S
sneaxiy 已提交
91
  ParallelExecutorPrivate *member_;
Q
Qiao Longfei 已提交
92
  std::vector<std::unique_ptr<ir::Graph>> async_graphs_;
93
};
Y
Yang Yang 已提交
94 95
}  // namespace framework
}  // namespace paddle