build_strategy.h 5.0 KB
Newer Older
Y
yuyang18 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

Y
yuyang18 已提交
17
#include <string>
18 19 20 21 22 23 24 25
#include <vector>

#include "paddle/fluid/framework/ir/pass_builder.h"
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/platform/device_context.h"
#include "paddle/fluid/platform/enforce.h"

P
peizhilin 已提交
26
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
27 28
#include "paddle/fluid/platform/nccl_helper.h"
#endif
Y
yuyang18 已提交
29

Y
yuyang18 已提交
30 31 32 33 34
namespace paddle {
namespace framework {
namespace details {

struct BuildStrategy {
C
chengduo 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
  // ParallelExecutor supports two modes of ReduceStrategy, kAllReduce and
  // kReduce, for CPU and GPU. If you use kAllReduce, different threads
  // optimize their parameters separately. If you use kReduce, the optimizations
  // of parameters are distributed to different threads.
  // For example, a model has 100 parameters and is running with four threads,
  // if you choose kAllReduce, every thread is to optimize 100 parameters
  // separately, if you choose kReduce, every thread is to optimize 25
  // parameters.
  // Of particular note is, if you use kReduce when using CPU training,
  // all the parameters are shared between different threads. This feature will
  // save memory.
  // FIXME(zcd): The result of the two modes(kAllReduce and kReduce) maybe not
  // equal for GPU. Because, the result of the different order of summing maybe
  // different, for example, the result of `a+b+c+d` may be different with the
  // result of `c+a+b+d`.
  // For GPU, the implementation of kAllReduce and kReduce is adopted NCCL,
  // so the result of kAllReduce and kReduce maybe not equal.
  // For CPU, if you want to fix the order of summing to make the result
  // of kAllReduce and kReduce no diff, you can add
  // `FLAGS_cpu_deterministic=true` to env.
Y
yuyang18 已提交
55 56 57 58 59 60 61 62
  enum class ReduceStrategy { kAllReduce = 0, kReduce = 1 };

  enum class GradientScaleStrategy {
    kCoeffNumDevice = 0,
    kOne = 1,
    kCustomized = 2,
  };

D
dzhwinter 已提交
63 64 65 66 67 68
  enum class OptimizeStrategy {
    // To be Implemented,bruteforce, recursive compute unused var names.
    kBruteForce = 0,
    kControlFlowGraph = 1,  // use cfg_graph algorithm, faster speed.
  };

Y
yuyang18 已提交
69
  ReduceStrategy reduce_{ReduceStrategy::kAllReduce};
Y
yuyang18 已提交
70
  GradientScaleStrategy gradient_scale_{GradientScaleStrategy::kCoeffNumDevice};
D
dzhwinter 已提交
71
  OptimizeStrategy strategy_{OptimizeStrategy::kControlFlowGraph};
Y
yuyang18 已提交
72 73

  std::string debug_graphviz_path_{""};
F
fengjiayi 已提交
74

C
chengduo 已提交
75 76
  bool fuse_elewise_add_act_ops_{false};

Y
yuyang18 已提交
77
  bool enable_data_balance_{false};
78

D
dzhwinter 已提交
79 80 81 82
  bool memory_optimize_{false};

  bool memory_early_delete_{false};

S
sneaxiy 已提交
83
  bool enable_sequential_execution_{false};
S
sneaxiy 已提交
84

85 86
  bool fuse_broadcast_op_{false};

87
  int num_trainers_{1};
88 89
  int trainer_id_{0};
  std::vector<std::string> trainers_endpoints_;
S
sneaxiy 已提交
90 91
  bool remove_unnecessary_lock_{false};

X
Xin Pan 已提交
92 93 94 95 96
  // NOTE:
  // Before you add new options, think if it's a general strategy that works
  // with other strategy. If not, the strategy should be created through
  // CreatePassesFromStrategy and the pass can be managed separately.

X
Xin Pan 已提交
97
  // User normally doesn't need to call this API.
X
Xin Pan 已提交
98
  // The PassBuilder allows for more customized insert, remove of passes
X
Xin Pan 已提交
99 100 101
  // from python side.
  // A new PassBuilder is created based on configs defined above and
  // passes are owned by the PassBuilder.
102
  std::shared_ptr<ir::PassBuilder> CreatePassesFromStrategy(
X
Xin Pan 已提交
103 104 105
      bool finalize_strategy) const;

  bool IsFinalized() const { return is_finalized_; }
106

X
Xin Pan 已提交
107 108
  // Apply the passes built by the pass_builder_. The passes will be
  // applied to the Program and output an ir::Graph.
109 110 111 112
  std::unique_ptr<ir::Graph> Apply(const ProgramDesc &main_program,
                                   const std::vector<platform::Place> &places,
                                   const std::string &loss_var_name,
                                   const std::vector<Scope *> &local_scopes,
113
                                   const size_t &nranks,
P
peizhilin 已提交
114
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
115 116
                                   const bool use_cuda,
                                   platform::NCCLContextMap *nccl_ctxs) const;
117
#else
118
                                   const bool use_cuda) const;
119 120
#endif

121 122 123 124 125 126 127
  // If set true, ParallelExecutor would build the main_program into multiple
  // graphs,
  // each of the graphs would run with one device. This approach can achieve
  // better performance
  // on some scenarios.
  mutable bool enable_parallel_graph_ = false;

128
 private:
X
Xin Pan 已提交
129
  mutable bool is_finalized_ = false;
130
  mutable std::shared_ptr<ir::PassBuilder> pass_builder_;
Y
yuyang18 已提交
131 132 133 134 135
};

}  // namespace details
}  // namespace framework
}  // namespace paddle