lstm_op.cc 9.9 KB
Newer Older
D
dangqingqing 已提交
1 2
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.

D
dangqingqing 已提交
3 4 5
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
D
dangqingqing 已提交
6

D
dangqingqing 已提交
7
http://www.apache.org/licenses/LICENSE-2.0
D
dangqingqing 已提交
8

D
dangqingqing 已提交
9 10 11 12 13
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dangqingqing 已提交
14

D
dangqingqing 已提交
15
#include "paddle/operators/lstm_op.h"
D
dangqingqing 已提交
16 17 18 19 20 21 22 23 24

namespace paddle {
namespace operators {

class LSTMOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

 protected:
25
  void InferShape(framework::InferShapeContext* ctx) const override {
D
dangqingqing 已提交
26 27 28 29
    PADDLE_ENFORCE(ctx->HasInput("Input"),
                   "Input(Input) of LSTM should not be null.");
    PADDLE_ENFORCE(ctx->HasOutput("Hidden"),
                   "Output(Hidden) of LSTM should not be null.");
30
    PADDLE_ENFORCE(ctx->HasOutput("Cell"),
D
dangqingqing 已提交
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
                   "Output(Cell) of LSTM should not be null.");

    auto x_dims = ctx->GetInputDim("Input");
    PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2.");

    if (ctx->HasInput("H0")) {
      PADDLE_ENFORCE(ctx->HasInput("C0"),
                     "Input(Cell) and Input(Hidden) of LSTM should not "
                     "be null at the same time.");
      auto h_dims = ctx->GetInputDim("H0");
      auto c_dims = ctx->GetInputDim("C0");
      PADDLE_ENFORCE(h_dims == c_dims,
                     "The dimension of Input(H0) and Input(C0) "
                     "should be the same.");
    }

47
    int frame_size = x_dims[1] / 4;
D
dangqingqing 已提交
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
    auto w_dims = ctx->GetInputDim("Weight");
    PADDLE_ENFORCE_EQ(w_dims.size(), 2,
                      "The rank of Input(Weight) should be 2.");
    PADDLE_ENFORCE_EQ(w_dims[0], frame_size,
                      "The first dimension of Input(Weight) "
                      "should be %d.",
                      frame_size);
    PADDLE_ENFORCE_EQ(w_dims[1], 4 * frame_size,
                      "The second dimension of Input(Weight) "
                      "should be 4 * %d.",
                      frame_size);
    auto b_dims = ctx->GetInputDim("Bias");
    PADDLE_ENFORCE_EQ(b_dims.size(), 2, "The rank of Input(Bias) should be 2.");
    PADDLE_ENFORCE_EQ(b_dims[0], 1,
                      "The first dimension of Input(Bias) should be 1.");
63
    if (ctx->Attrs().Get<bool>("usePeepholes")) {
D
dangqingqing 已提交
64 65 66 67 68 69 70
      PADDLE_ENFORCE_EQ(b_dims[1], 7 * frame_size,
                        "The second dimension of Input(Bias) should be "
                        "7 * %d if enable peepholes connection",
                        frame_size);
    } else {
      PADDLE_ENFORCE_EQ(b_dims[1], 4 * frame_size,
                        "The second dimension of Input(Bias) should be "
Y
Yu Yang 已提交
71
                        "4 * %d if disable peepholes connection",
D
dangqingqing 已提交
72 73
                        frame_size);
    }
74 75 76
    ctx->SetOutputDim("Hidden", {x_dims[0], frame_size});
    ctx->SetOutputDim("Cell", {x_dims[0], frame_size});
    ctx->SetOutputDim("BatchGate", x_dims);
D
dangqingqing 已提交
77 78 79 80 81 82 83 84 85 86 87 88
    ctx->ShareLoD("Input", "Hidden");
    ctx->ShareLoD("Input", "Cell");
  }
};

class LSTMOpMaker : public framework::OpProtoAndCheckerMaker {
 public:
  LSTMOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker)
      : OpProtoAndCheckerMaker(proto, op_checker) {
    AddInput("Input",
             "(LoDTensor) the first input is a LodTensor, which support "
             "variable-time length input sequence. The underlying tensor in "
Y
Yu Yang 已提交
89
             "this LoDTensor is a matrix with shape (T X 4D), where, T is the "
D
dangqingqing 已提交
90 91 92 93 94 95 96 97 98 99 100
             "total time steps in this mini-batch, D is the hidden size.");
    AddInput("H0",
             "(Tensor, optional) the initial hidden state is an optional "
             "input. This is a tensor with shape (N x D), where N is the "
             "batch size, D is the hidden size.");
    AddInput("C0",
             "(Tensor, optional) the initial cell state is an optional "
             "input. This is a tensor with shape (N x D), where N is the "
             "batch size. `H0` and `C0` can be NULL but only at the same time");
    AddInput("Weight",
             "(Tensor) the learnable hidden-hidden weights."
D
dangqingqing 已提交
101 102
             " - The shape is (D x 4D), where D is the hidden size. "
             " - Weight = {W_ch, W_ih, W_fh, W_oh}");
D
dangqingqing 已提交
103 104 105
    AddInput("Bias",
             "(Tensor) the learnable weights, which contains two parts: "
             "input-hidden bias weight and peephole connections weight if "
D
dangqingqing 已提交
106
             "setting `usePeepholes` True. "
107
             "1. `usePeepholes = False` "
D
dangqingqing 已提交
108 109
             " - The shape is (1 x 4D). "
             " - Bias = {b_c, b_i, b_f, b_o}."
110
             "2. `usePeepholes = True` "
D
dangqingqing 已提交
111 112
             " - The shape is (1 x 7D). "
             " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}.");
113 114
    AddOutput("BatchGate",
              "(LoDTensor) This LoDTensor contains input gate, forget gate "
Y
Yu Yang 已提交
115
              "and output gate after the nonlinear computation. This "
116 117 118 119 120
              "LoDTensor has the same shape with the reorganized input, which "
              "was also be called batch input. The LoD size is 2. The first "
              "LoD is the batch offsets and the second LoD contains the "
              "indexes, which denote the position of reorganized sequence "
              "in the raw input.")
D
dangqingqing 已提交
121
        .AsIntermediate();
D
dangqingqing 已提交
122 123 124 125 126 127
    AddOutput("Hidden",
              "(LoDTensor) the hidden state lod tensor of LSTM operator. "
              "The shape and lod is the same with the `Input`.");
    AddOutput("Cell",
              "(LoDTensor) the cell state lod tensor of LSTM operator. "
              "The shape and lod is the same with the `Input`.");
128
    AddAttr<bool>("usePeepholes",
D
dangqingqing 已提交
129 130 131
                  "(bool, defalut: True) "
                  "whether to enable diagonal/peephole connections.")
        .SetDefault(true);
132
    AddAttr<bool>("isReverse",
D
dangqingqing 已提交
133 134
                  "(bool, defalut: False) "
                  "whether to compute reversed LSTM.")
135
        .SetDefault(false);
D
dangqingqing 已提交
136
    AddAttr<std::string>(
137
        "gateActivation",
Y
Yu Yang 已提交
138
        "(string, default: sigmoid)"
D
dangqingqing 已提交
139
        "The activation for input gate, forget gate and output "
Y
Yu Yang 已提交
140
        "gate, `sigmoid` by default.")
D
dangqingqing 已提交
141
        .SetDefault("sigmoid");
142
    AddAttr<std::string>("cellActivation",
Y
Yu Yang 已提交
143
                         "(string, default: tanh)"
D
dangqingqing 已提交
144 145
                         "The activation for cell output, `tanh` by defalut.")
        .SetDefault("tanh");
146
    AddAttr<std::string>("candidateActivation",
Y
Yu Yang 已提交
147
                         "(string, default: tanh)"
D
dangqingqing 已提交
148
                         "The activation for candidate hidden state, "
Y
Yu Yang 已提交
149
                         "`tanh` by default.")
D
dangqingqing 已提交
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
        .SetDefault("tanh");
    AddComment(R"DOC(Long-Short Term Memory (LSTM) Operator

The defalut implementation is diagonal/peephole connection [1], the formula is
as follows

    i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i)

    f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f)

    \tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c)

    o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o)

    c_t = f_t ⊙ c_{t-1} + i_t ⊙ \tilde{c_t}

    h_t = o_t ⊙ act_h(c_t)

where the W terms denote weight matrices (e.g. \f$W_{xi}\f$ is the matrix
of weights from the input gate to the input), \f$W_{ic}, W_{fc}, W_{oc}\f$
are diagonal weight matrices for peephole connections. In our implenmention,
We use vectors to reprenset these diagonal weight matrices. The b terms
denote bias vectors (\f$b_i\f$ is the input gate bias vector), \f$\sigma\f$
is the non-line actications, such as logistic sigmoid function, and
\f$i, f, o\f$ and \f$c\f$ are respectively the input gate, forget gate,
output gate and cell activation vectors, all of which are the same size as
the cell output activation vector \f$h\f$.

The ⊙ is the element-wise product of the vectors, \f$act_g\f$ and \f$act_h\f$
are the cell input and cell output activation functions, `tanh` is usually
used for them. \f$\tilde{c_t}\f$ is also called candidate hidden state,
which is computed based on the current input and the previous hidden state.

183
Set `usePeepholes` False to disable peephole connection [2]. The formula
D
dangqingqing 已提交
184 185 186
is omitted here.

@note These \f$W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}\f$
D
dangqingqing 已提交
187 188
operations on the input x_{t} were NOT included in this operator.
Users can choose to use fully-connect operator before LSTM operator.
D
dangqingqing 已提交
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205

[1] Hasim Sak, Andrew Senior, and Francoise Beaufays. Long short-term memory
recurrent neural network architectures for large scale acoustic modeling.
INTERSPEECH, 2014.

[2] S. Hochreiter and J. Schmidhuber. Long Short-Term Memory.
Neural Computation, 9(8):1735-1780, 1997.

)DOC");
  }
};

class LSTMGradOp : public framework::OperatorWithKernel {
 public:
  using framework::OperatorWithKernel::OperatorWithKernel;

 protected:
206
  void InferShape(framework::InferShapeContext* ctx) const override {
D
dangqingqing 已提交
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Hidden")),
                   "Input(Hidden@GRAD) should not be null");
    PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Cell")),
                   "Input(Cell@GRAD) should not be null");
    ctx->SetOutputDim(framework::GradVarName("Weight"),
                      ctx->GetInputDim("Weight"));
    ctx->SetOutputDim(framework::GradVarName("Bias"), ctx->GetInputDim("Bias"));
  }
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;
REGISTER_OP(lstm, ops::LSTMOp, ops::LSTMOpMaker, lstm_grad, ops::LSTMGradOp);
REGISTER_OP_CPU_KERNEL(lstm, ops::LSTMKernel<paddle::platform::CPUPlace, float>,
                       ops::LSTMKernel<paddle::platform::CPUPlace, double>);
REGISTER_OP_CPU_KERNEL(lstm_grad,
                       ops::LSTMGradKernel<paddle::platform::CPUPlace, float>,
                       ops::LSTMGradKernel<paddle::platform::CPUPlace, double>);