lstmp_op.cc 14.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "paddle/operators/lstmp_op.h"

namespace paddle {
namespace operators {

class LSTMPOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    PADDLE_ENFORCE(ctx->HasInput("Input"),
26
                   "Input(Input) of LSTMP operator should not be null.");
27
    PADDLE_ENFORCE(ctx->HasInput("Weight"),
28
                   "Input(Weight) of LSTMP operator should not be null.");
29
    PADDLE_ENFORCE(ctx->HasInput("ProjWeight"),
30
                   "Input(ProjWeight) of LSTMP operator should not be null.");
31
    PADDLE_ENFORCE(ctx->HasInput("Bias"),
32
                   "Input(Bias) of LSTMP operator should not be null.");
33 34

    PADDLE_ENFORCE(ctx->HasOutput("Projection"),
35
                   "Output(Projection) of LSTMP operator should not be null.");
36
    PADDLE_ENFORCE(ctx->HasOutput("Cell"),
37
                   "Output(Cell) of LSTMP operator should not be null.");
38
    PADDLE_ENFORCE(ctx->HasOutput("BatchGate"),
39
                   "Output(BatchGate) of LSTMP operator should not be null.");
40
    PADDLE_ENFORCE(ctx->HasOutput("BatchCellPreAct"),
41 42
                   "Output(BatchCellPreAct) of LSTMP operator should not be "
                   "null.");
43
    PADDLE_ENFORCE(ctx->HasOutput("BatchHidden"),
44
                   "Output(BatchHidden) of LSTMP operator should not be null.");
45 46

    auto in_dims = ctx->GetInputDim("Input");
47 48
    PADDLE_ENFORCE_EQ(in_dims.size(), 2,
                      "Input(X)'s rank of LSTMP operator must be 2.");
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70

    int frame_size = in_dims[1] / 4;
    auto w_dims = ctx->GetInputDim("Weight");
    auto proj_dims = ctx->GetInputDim("ProjWeight");
    PADDLE_ENFORCE_EQ(w_dims.size(), 2,
                      "The rank of Input(Weight) should be 2.");
    PADDLE_ENFORCE_EQ(w_dims[0], proj_dims[1],
                      "The first dimension of Input(Weight) "
                      "should be %d.",
                      proj_dims[1]);
    PADDLE_ENFORCE_EQ(w_dims[1], 4 * frame_size,
                      "The second dimension of Input(Weight) "
                      "should be 4 * %d.",
                      frame_size);

    PADDLE_ENFORCE_EQ(proj_dims.size(), 2,
                      "The rank of Input(ProjWeight) should be 2.");
    PADDLE_ENFORCE_EQ(proj_dims[0], frame_size,
                      "The first dimension of Input(ProjWeight) "
                      "should be %d.",
                      frame_size);

71 72
    if (ctx->HasInput("H0")) {
      PADDLE_ENFORCE(ctx->HasInput("C0"),
73 74
                     "Input(C0) of LSTMP operator should not be null after "
                     "Input(H0) provided.");
75 76 77 78 79 80 81 82
      auto h_dims = ctx->GetInputDim("H0");
      auto c_dims = ctx->GetInputDim("C0");
      PADDLE_ENFORCE(h_dims == c_dims,
                     "The dimension of Input(H0) and Input(C0) "
                     "should be the same.");
      ctx->SetOutputDim("OrderedP0", {h_dims[0], proj_dims[1]});
    }

83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
    auto b_dims = ctx->GetInputDim("Bias");
    PADDLE_ENFORCE_EQ(b_dims.size(), 2, "The rank of Input(Bias) should be 2.");
    PADDLE_ENFORCE_EQ(b_dims[0], 1,
                      "The first dimension of Input(Bias) should be 1.");

    if (ctx->Attrs().Get<bool>("use_peepholes")) {
      PADDLE_ENFORCE_EQ(b_dims[1], 7 * frame_size,
                        "The second dimension of Input(Bias) should be "
                        "7 * %d if enable peepholes connection",
                        frame_size);
    } else {
      PADDLE_ENFORCE_EQ(b_dims[1], 4 * frame_size,
                        "The second dimension of Input(Bias) should be "
                        "4 * %d if disable peepholes connection",
                        frame_size);
    }

    framework::DDim out_dims({in_dims[0], frame_size});
    framework::DDim proj_out_dims({in_dims[0], proj_dims[1]});
    ctx->SetOutputDim("Projection", proj_out_dims);
    ctx->SetOutputDim("Cell", out_dims);
    ctx->SetOutputDim("BatchGate", in_dims);
    ctx->SetOutputDim("BatchCellPreAct", out_dims);
106
    ctx->SetOutputDim("BatchHidden", out_dims);
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
    ctx->ShareLoD("Input", "Projection");
    ctx->ShareLoD("Input", "Cell");
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(
        framework::ToDataType(ctx.Input<framework::LoDTensor>("Input")->type()),
        ctx.device_context());
  }
};

class LSTMPOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  LSTMPOpMaker(OpProto* proto, OpAttrChecker* op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("Input",
Y
Yibing Liu 已提交
125
             "(LoDTensor) the input for sequence data, which supports "
126 127 128 129 130 131 132 133 134 135 136
             "variable-time length input sequence. The underlying tensor in "
             "this LoDTensor is a matrix with shape (T X 4D), where T is the "
             "total time steps in this mini-batch, D is the hidden size.");
    AddInput("H0",
             "(Tensor, optional) the initial hidden state is an optional "
             "input. This is a tensor with shape (N x D), where N is the "
             "batch size and D is the hidden size.")
        .AsDispensable();
    AddInput("C0",
             "(Tensor, optional) the initial cell state is an optional "
             "input. This is a tensor with shape (N x D), where N is the "
137
             "batch size. `C0` should not be null if `H0` provided.")
138 139 140
        .AsDispensable();
    AddInput("Weight",
             "(Tensor) the learnable hidden-hidden weights."
Y
Yibing Liu 已提交
141 142
             " - The shape is (P x 4D), where P is the projection layer size "
             "and  D is the hidden size."
143 144
             " - Weight = {W_cr, W_ir, W_fr, W_or}");
    AddInput("ProjWeight",
Y
Yibing Liu 已提交
145
             "(Tensor) the learnable weight of the projection layer."
146
             " - The shape is (D x P), where P is the recurrent projection "
Y
Yibing Liu 已提交
147 148
             "layer size and  D is the hidden size."
             " - ProjWeight = {W_rh}");
149
    AddInput("Bias",
Y
Yibing Liu 已提交
150 151 152
             "(Tensor) the learnable biases, which contains two parts: "
             "input-hidden biases and peephole connections weights if "
             "setting `use_peepholes` to `True`. "
153 154 155 156 157 158 159 160
             "1. `use_peepholes = False` "
             " - The shape is (1 x 4D). "
             " - Bias = {b_c, b_i, b_f, b_o}."
             "2. `use_peepholes = True` "
             " - The shape is (1 x 7D). "
             " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}.");
    AddOutput("Projection",
              "(LoDTensor) the projection of the hidden state of LSTMP "
Y
Yibing Liu 已提交
161
              "operator. The shape is (T x P), and LoD is the same with the "
162 163 164 165 166 167
              "`Input`.");
    AddOutput("Cell",
              "(LoDTensor) the cell state of LSTMP operator. "
              "The shape is (T x D), and lod is the same with the `Input`.");
    AddOutput("BatchGate",
              "(LoDTensor) This LoDTensor contains input gate, forget gate "
Y
Yibing Liu 已提交
168 169 170 171 172
              "and output gate after the activations. This LoDTensor has the "
              "same shape as the reorganized input, which is also be called "
              "batch input. The LoD size is 2. The first-level LoD is the "
              "batch offsets and the second contains the indices, which "
              "denotes the position of reorganized sequence in the raw input.")
173 174
        .AsIntermediate();
    AddOutput("BatchCellPreAct",
Y
Yibing Liu 已提交
175 176 177
              "(LoDTensor) the pre-activation cell state reorganized in batch. "
              "This LoDTensor is obtained in the forward and used in the "
              "backward.")
178
        .AsIntermediate();
179
    AddOutput("BatchHidden",
Y
Yibing Liu 已提交
180 181 182
              "(LoDTensor) the hidden state reorganized in batch. "
              "This LoDTensor is obtained in the forward and used in the "
              "backward.")
183 184 185 186 187 188
        .AsIntermediate();
    AddOutput("OrderedP0",
              "(Tensor) the projection of the initial hidden state "
              "H0. This is a tensor with shape (N x P), where N is the "
              "batch size and P is the hidden size.")
        .AsIntermediate();
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
    AddAttr<bool>("use_peepholes",
                  "(bool, defalut: True) "
                  "whether to enable diagonal/peephole connections.")
        .SetDefault(true);
    AddAttr<bool>("is_reverse",
                  "(bool, defalut: False) "
                  "whether to compute reversed LSTMP.")
        .SetDefault(false);
    AddAttr<std::string>(
        "gate_activation",
        "(string, default: sigmoid)"
        "The activation for input gate, forget gate and output "
        "gate, `sigmoid` by default.")
        .SetDefault("sigmoid")
        .InEnum({"sigmoid", "tanh", "relu", "identity"});
    AddAttr<std::string>("cell_activation",
                         "(string, default: tanh)"
                         "The activation for cell output, `tanh` by defalut.")
        .SetDefault("tanh")
        .InEnum({"sigmoid", "tanh", "relu", "identity"});
    AddAttr<std::string>("candidate_activation",
                         "(string, default: tanh)"
                         "The activation for candidate hidden state, "
                         "`tanh` by default.")
        .SetDefault("tanh")
        .InEnum({"sigmoid", "tanh", "relu", "identity"});
215 216 217 218 219 220
    AddAttr<std::string>("proj_activation",
                         "(string, default: tanh)"
                         "The activation for projection output, "
                         "`tanh` by defalut.")
        .SetDefault("tanh")
        .InEnum({"sigmoid", "tanh", "relu", "identity"});
221
    AddComment(R"DOC(
Y
Yibing Liu 已提交
222
Long-Short Term Memory with recurrent Projection layer (LSTMP) Operator.
223

Y
Yibing Liu 已提交
224 225 226 227 228
LSTMP has a separate projection layer after the LSTM layer, projecting the 
original hidden state to a lower-dimensional one, which is proposed to reduce 
the number of total parameters and furthermore computational complexity for 
the LSTM, espeacially for the case that the size of output units is relative 
large (https://research.google.com/pubs/archive/43905.pdf). 
229

230 231 232
The formula is as follows:

$$
233
i_t = \sigma(W_{ix}x_{t} + W_{ir}r_{t-1} + W_{ic}c_{t-1} + b_i) \\
234

235
f_t = \sigma(W_{fx}x_{t} + W_{fr}r_{t-1} + W_{fc}c_{t-1} + b_f) \\
236

237
\tilde{c_t} = act_g(W_{cx}x_t + W_{cr}r_{t-1} + b_c) \\
238

239
o_t = \sigma(W_{ox}x_{t} + W_{or}r_{t-1} + W_{oc}c_t + b_o) \\
240

241
c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\
Y
Yibing Liu 已提交
242

243
h_t = o_t \odot act_h(c_t) \\
244

Y
Yibing Liu 已提交
245
r_t = \overline{act_h}(W_{rh}h_t)
246 247 248 249 250 251 252
$$

where the W terms denote weight matrices (e.g. $W_{xi}$ is the matrix
of weights from the input gate to the input), $W_{ic}, W_{fc}, W_{oc}$
are diagonal weight matrices for peephole connections. In our implementation,
we use vectors to reprenset these diagonal weight matrices. The b terms
denote bias vectors ($b_i$ is the input gate bias vector), $\sigma$
Y
Yibing Liu 已提交
253
is the activation, such as logistic sigmoid function, and
254 255
$i, f, o$ and $c$ are the input gate, forget gate, output gate,
and cell activation vectors, respectively, all of which have the same size as
Y
Yibing Liu 已提交
256 257 258 259
the cell output activation vector $h$. Here $h$ is usually called the hidden 
state and $r$ denotes its recurrent projection. And $\tilde{c_t}$ is also 
called the candidate hidden state, whose computation is based on the current 
input and previous hidden state.
260 261 262

The $\odot$ is the element-wise product of the vectors. $act_g$ and $act_h$
are the cell input and cell output activation functions and `tanh` is usually
263 264
used for them. $\overline{act_h}$ is the activation function for the 
projection output, usually using `identity` or same as $act_h$.
265 266 267

Note that these $W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}$
operations on the input $x_{t}$ are NOT included in this operator.
Y
Yibing Liu 已提交
268
Users can choose to use fully-connected operator before LSTMP operator.
269 270 271 272 273 274 275 276 277 278 279

)DOC");
  }
};

class LSTMPGradOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

  void InferShape(framework::InferShapeContext* ctx) const override {
    PADDLE_ENFORCE(ctx->HasInput("Input"),
280
                   "Input(Input) of LSTMP operator should not be null.");
281
    PADDLE_ENFORCE(ctx->HasInput("Projection"),
282
                   "Input(Projection) of LSTMP operator should not be null.");
283
    PADDLE_ENFORCE(ctx->HasInput("Cell"),
284
                   "Input(Cell) of LSTMP operator should not be null.");
285
    PADDLE_ENFORCE(ctx->HasInput("Weight"),
286
                   "Input(Weight) of LSTMP operator should not be null.");
287
    PADDLE_ENFORCE(ctx->HasInput("ProjWeight"),
288
                   "Input(ProjWeight) of LSTMP operator should not be null.");
289
    PADDLE_ENFORCE(ctx->HasInput("Bias"),
290
                   "Input(Bias) of LSTMP operator should not be null.");
291 292

    PADDLE_ENFORCE(ctx->HasInput("BatchGate"),
293
                   "Input(BatchGate) of LSTMP operator should not be null.");
294
    PADDLE_ENFORCE(ctx->HasInput("BatchCellPreAct"),
295
                   "Input(BatchGate) of LSTMP operator should not be null.");
296 297 298 299 300 301 302 303 304

    auto SetOutGradDim = [&ctx](const std::string& name) {
      auto g_name = framework::GradVarName(name);
      if (ctx->HasOutput(g_name))
        ctx->SetOutputDim(g_name, ctx->GetInputDim(name));
    };

    SetOutGradDim("Input");
    SetOutGradDim("Weight");
305
    SetOutGradDim("ProjWeight");
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
    SetOutGradDim("Bias");
    SetOutGradDim("H0");
    SetOutGradDim("C0");
  }

 protected:
  framework::OpKernelType GetExpectedKernelType(
      const framework::ExecutionContext& ctx) const override {
    return framework::OpKernelType(
        framework::ToDataType(ctx.Input<framework::LoDTensor>("Input")->type()),
        ctx.device_context());
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP(lstmp, ops::LSTMPOp, ops::LSTMPOpMaker, lstmp_grad,
            ops::LSTMPGradOp);
REGISTER_OP_CPU_KERNEL(
    lstmp, ops::LSTMPKernel<paddle::platform::CPUDeviceContext, float>,
    ops::LSTMPKernel<paddle::platform::CPUDeviceContext, double>);
REGISTER_OP_CPU_KERNEL(
    lstmp_grad, ops::LSTMPGradKernel<paddle::platform::CPUDeviceContext, float>,
    ops::LSTMPGradKernel<paddle::platform::CPUDeviceContext, double>);