GradientMachine.h 7.2 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Z
zhangjinchao01 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#pragma once

#include <iostream>
#include <vector>

#include "ModelConfig.pb.h"
Y
Yu Yang 已提交
21
#include "TrainerConfig.pb.h"
X
Xin Pan 已提交
22 23
#include "paddle/legacy/gserver/dataproviders/DataProvider.h"
#include "paddle/legacy/gserver/layers/Layer.h"
X
Xin Pan 已提交
24 25 26
#include "paddle/legacy/math/Matrix.h"
#include "paddle/legacy/parameter/Parameter.h"
#include "paddle/legacy/parameter/ParameterUpdaterBase.h"
Y
Yu Yang 已提交
27
#include "paddle/utils/Thread.h"
Z
zhangjinchao01 已提交
28

29
#ifndef PADDLE_MOBILE_INFERENCE
X
Xin Pan 已提交
30
#include "paddle/legacy/gserver/evaluators/Evaluator.h"
31 32
#endif

Z
zhangjinchao01 已提交
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
namespace paddle {
/**
 * @brief A gradient machine is capable of calculating some outputs given
 *        some inputs and performing gradient calculation based on the
 *        derivative from the outputs.
 *
 * A gradient machine can be either a full neural network or part of a neural
 * network.
 *
 * Usage for training:
 *
 *  1. Prepare inArgs. Put your input data into inArgs[i].value.
 *
 *  2. Call forward(inArgs, &outArgs)
 *
 *  3. Calculate gradient with respect to outArgs[i]->value
 *     and fill them into outArgs[i]->grad.
 *     This step can be skipped if your the outputs are from cost layers.
 *
 *  4. Call backward(). After backward, gradient of each parameter is
 *     accumulated to getParameters()[i]->getBuf(PARAMETER_GRADIENT)
 *
 *  5. Update parameter value getParameters()[i]->getBuf(PARAMETER_VALUE) using
 *     gradients.
 *
 *  6. Clear gradients to zero.
 *
 * Usage for prediction:
 *
 *  1. Prepare inArgs. Put your input data into inArgs[i].value.
 *
 *  2. Call forward(inArgs, &outArgs)
 *
 *  3. Obtain the prediction result from outArgs[i]
 */

typedef std::vector<LayerStatePtr> MachineState;

class GradientMachine;

typedef std::shared_ptr<GradientMachine> GradientMachinePtr;

class GradientMachine {
W
Wu Yi 已提交
76
 public:
Z
zhangjinchao01 已提交
77 78 79 80 81 82 83 84 85 86 87 88
  enum CreateMode {
    kNormal = 0,
    kSgdSparseCpuTraining = 3,
    kTesting = 4,
    kCustom = 10
  };

  /**
   * Create a gradient machine from ModelConfig
   * Parameter will have parameterTypes
   */
  static GradientMachine* create(
89 90
      const ModelConfig& config,
      int mode = kNormal,
Z
zhangjinchao01 已提交
91
      const std::vector<ParameterType>& parameterTypes =
92 93
          std::vector<ParameterType>{
              PARAMETER_VALUE, PARAMETER_GRADIENT, PARAMETER_MOMENTUM});
Z
zhangjinchao01 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109

  virtual ~GradientMachine() {}

  /**
   * Prefetch row ids of sparse parameter.
   */
  virtual void prefetch(const std::vector<Argument>& inArgs) { (void)inArgs; }

  /**
   * @brief Forward propagation.
   *
   * Calculate outputs (outArgs) based the inputs (inArgs)
   *
   * @note: if passType==PASS_TEST, then backward() should not be called
   */
  virtual void forward(const std::vector<Argument>& inArgs,
110 111
                       std::vector<Argument>* outArgs,
                       PassType passType) = 0;
Z
zhangjinchao01 已提交
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139

  /**
   * @brief Backward propagation.
   *
   * Calculate the gradient of inArgs and parameter.
   *
   * This function should only be called after a corresponding forward() call.
   * The caller is responsible for filling the correct grad for the outArgs
   * obtained using forward().
   *
   * It may also change the grad field for the inArgs supplied at forward()
   */
  virtual void backward(const UpdateCallback& callback = nullptr) = 0;

  /**
   * Combine forward() and backward(). For multithread training, this
   * may be faster.
   *
   * @note: passType PASS_TEST is not allowed for forwardBackward().
   */
  virtual void forwardBackward(const std::vector<Argument>& inArgs,
                               std::vector<Argument>* outArgs,
                               PassType passType,
                               const UpdateCallback& callback = nullptr) {
    forward(inArgs, outArgs, passType);
    backward(callback);
  }

Q
qijun 已提交
140
  virtual Argument getLayerOutput(const std::string& layerName) = 0;
141

Z
zhangjinchao01 已提交
142 143 144 145 146 147 148 149 150 151 152
  // see comment in Layer.h for the function with the same name
  virtual void resetState() {}

  // set machine state
  virtual void setState(const MachineState& machineState) {}

  // save machine state
  virtual void getState(MachineState& machineState) {}

  virtual void onPassEnd() = 0;

153
#ifndef PADDLE_MOBILE_INFERENCE
Z
zhangjinchao01 已提交
154 155 156
  /**
   * Create an evaluator which can be used for eval()
   */
Y
Yu Yang 已提交
157
  virtual Evaluator* makeEvaluator() const = 0;
Z
zhangjinchao01 已提交
158 159 160 161

  /**
   * evaluate using the given evaluator
   */
Y
Yu Yang 已提交
162
  virtual void eval(Evaluator* evaluator) const = 0;
163
#endif
Z
zhangjinchao01 已提交
164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188

  std::vector<ParameterPtr>& getParameters() { return parameters_; }

  std::vector<ParameterPtr>& getNonStaticParameters() {
    if (nonStaticParameters_.empty()) {
      for (auto para : parameters_) {
        if (!para->isStatic()) {
          nonStaticParameters_.push_back(para);
        }
      }
    }
    return nonStaticParameters_;
  }

  inline bool hasStaticParameters() {
    return parameters_.size() != getNonStaticParameters().size();
  }

  /**
   * @brief   Used before formal training, start work-threads and set
   *          trainer Parameters;
   *
   * @note    This function will only been implemented and used in a
   *          multithreaded environment.
   */
189
  virtual void start() {}
Z
zhangjinchao01 已提交
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219

  /**
   * @brief   check  each work-thread whether is failed/error/finish,
   *          if not, return ture, and yes return false.
   *
   * @note    This function will only been implemented and used in a
   *          multithreaded environment.
   */
  virtual void finish() {}

  /**
   * @brief   set the training status a "finished" value, the sub_work_threads
   *          will option the change, and then exit.
   *
   * @note    This function will only been implemented and used in a
   *          multithreaded environment.
   */
  virtual bool trainIsOn() { return true; }

  /**
   * @brief   when all or some of the sub-workThreads are suspended to waiting
   *          controller's instructions, and after some processing done in the
   *          controller, it will call this function to wake up all the pending
   *          thread.
   *
   * @note    This function will only been implemented and used in a
   *          multithreaded environment.
   */
  virtual void restart() {}

220
  /// Set the gradient of the output from outside.
Z
zhangjinchao01 已提交
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
  virtual void setOutputGrad(const std::vector<Argument>& args) {
    LOG(FATAL) << "Not implemented!";
  }

  void saveParameters(const std::string& dir) const;

  void loadParameters(const std::string& dir);

  void randParameters();

  virtual void getStats(real& cost, int64_t& numProcessed) {
    (void)cost;
    (void)numProcessed;
  }

236 237 238 239 240 241 242
  /**
   * @brief   Release the middle layer's output memory.
   *
   * @note    This function is used for memory optimization in inference.
   */
  virtual void releaseOutput() {}

W
Wu Yi 已提交
243
 protected:
Z
zhangjinchao01 已提交
244 245 246 247 248 249 250
  virtual void onLoadParameter() {}

  std::vector<ParameterPtr> parameters_;
  std::vector<ParameterPtr> nonStaticParameters_;
};

}  // namespace paddle