未验证 提交 f6e8bf0d 编写于 作者: X Xing Wu 提交者: GitHub

update cudnn rnn weights, test=develop (#23929)

上级 720d1899
......@@ -39,7 +39,6 @@ class LSTMCell(Layer):
\\tilde{c_t} &= tanh(W_{cx}x_t + W_{ch}h_{t-1} + b_c)
c_t &= f_t \\odot c_{t-1} + i_t \\odot \\tilde{c_t}
h_t &= o_t \\odot tanh(c_t)
Args:
hidden_size (integer): The hidden size used in the Cell.
input_size (integer): The input size used in the Cell.
......@@ -64,30 +63,25 @@ class LSTMCell(Layer):
Returns:
None
Examples:
.. code-block:: python
from paddle import fluid
import paddle.fluid.core as core
from paddle.fluid.dygraph.rnn import LSTMCell
import numpy as np
batch_size = 64
input_size = 128
hidden_size = 256
step_input_np = np.random.uniform(-0.1, 0.1, (
batch_size, input_size)).astype('float64')
pre_hidden_np = np.random.uniform(-0.1, 0.1, (
batch_size, hidden_size)).astype('float64')
pre_cell_np = np.random.uniform(-0.1, 0.1, (
batch_size, hidden_size)).astype('float64')
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
else:
place = core.CPUPlace()
with fluid.dygraph.guard(place):
cudnn_lstm = LSTMCell(hidden_size, input_size)
step_input_var = fluid.dygraph.to_variable(step_input_np)
......@@ -139,12 +133,12 @@ class LSTMCell(Layer):
self._weight_ih = self.create_parameter(
attr=weight_ih_param_attr,
shape=[self._input_size, 4 * self._hidden_size],
shape=[4 * self._hidden_size, self._input_size],
dtype=self._dtype)
self._weight_hh = self.create_parameter(
attr=weight_hh_param_attr,
shape=[self._hidden_size, 4 * self._hidden_size],
shape=[4 * self._hidden_size, self._hidden_size],
dtype=self._dtype)
self._bias_ih = self.create_parameter(
......@@ -180,10 +174,10 @@ class LSTMCell(Layer):
def forward(self, input, pre_hidden, pre_cell):
if self._use_cudnn_impl:
igates = layers.matmul(input, y=self._weight_ih)
igates = layers.matmul(input, y=self._weight_ih, transpose_y=True)
igates = layers.elementwise_add(igates, self._bias_ih)
hgates = layers.matmul(pre_hidden, self._weight_hh)
hgates = layers.matmul(
pre_hidden, self._weight_hh, transpose_y=True)
hgates = layers.elementwise_add(hgates, self._bias_hh)
chunked_igates = layers.split(igates, num_or_sections=4, dim=1)
......@@ -264,28 +258,23 @@ class GRUCell(Layer):
Returns:
None
Examples:
.. code-block:: python
from paddle import fluid
import paddle.fluid.core as core
from paddle.fluid.dygraph.rnn import GRUCell
import numpy as np
batch_size = 64
input_size = 128
hidden_size = 256
step_input_np = np.random.uniform(-0.1, 0.1, (
batch_size, input_size)).astype('float64')
pre_hidden_np = np.random.uniform(-0.1, 0.1, (
batch_size, hidden_size)).astype('float64')
if core.is_compiled_with_cuda():
place = core.CUDAPlace(0)
else:
place = core.CPUPlace()
with fluid.dygraph.guard(place):
cudnn_gru = GRUCell(hidden_size, input_size)
step_input_var = fluid.dygraph.to_variable(step_input_np)
......@@ -334,12 +323,12 @@ class GRUCell(Layer):
self._weight_ih = self.create_parameter(
attr=weight_ih_param_attr,
shape=[self._input_size, 3 * self._hidden_size],
shape=[3 * self._hidden_size, self._input_size],
dtype=self._dtype)
self._weight_hh = self.create_parameter(
attr=weight_hh_param_attr,
shape=[self._hidden_size, 3 * self._hidden_size],
shape=[3 * self._hidden_size, self._hidden_size],
dtype=self._dtype)
self._bias_ih = self.create_parameter(
......@@ -402,9 +391,10 @@ class GRUCell(Layer):
if self._use_cudnn_impl:
igates = layers.matmul(input, y=self._weight_ih)
igates = layers.matmul(input, y=self._weight_ih, transpose_y=True)
igates = layers.elementwise_add(igates, self._bias_ih)
hgates = layers.matmul(pre_hidden, self._weight_hh)
hgates = layers.matmul(
pre_hidden, self._weight_hh, transpose_y=True)
hgates = layers.elementwise_add(hgates, self._bias_hh)
chunked_igates = layers.split(igates, num_or_sections=3, dim=1)
......
......@@ -34,9 +34,9 @@ def tanh(x):
def cudnn_step(step_input_np, pre_hidden_np, weight_ih, bias_ih, weight_hh,
bias_hh):
igates = np.matmul(step_input_np, weight_ih)
igates = np.matmul(step_input_np, weight_ih.transpose(1, 0))
igates += bias_ih
hgates = np.matmul(pre_hidden_np, weight_hh)
hgates = np.matmul(pre_hidden_np, weight_hh.transpose(1, 0))
hgates += bias_hh
chunked_igates = np.split(igates, indices_or_sections=3, axis=1)
......
......@@ -32,7 +32,12 @@ def tanh(x):
return 2. * sigmoid(2. * x) - 1.
def cudnn_step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0):
def non_cudnn_step(step_in,
pre_hidden,
pre_cell,
gate_w,
gate_b,
forget_bias=1.0):
concat_1 = np.concatenate([step_in, pre_hidden], 1)
gate_input = np.matmul(concat_1, gate_w)
......@@ -45,12 +50,12 @@ def cudnn_step(step_in, pre_hidden, pre_cell, gate_w, gate_b, forget_bias=1.0):
return new_hidden, new_cell
def non_cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih,
bias_ih, weight_hh, bias_hh):
def cudnn_step(step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih,
weight_hh, bias_hh):
igates = np.matmul(step_input_np, weight_ih)
igates = np.matmul(step_input_np, weight_ih.transpose(1, 0))
igates = igates + bias_ih
hgates = np.matmul(pre_hidden_np, weight_hh)
hgates = np.matmul(pre_hidden_np, weight_hh.transpose(1, 0))
hgates = hgates + bias_hh
chunked_igates = np.split(igates, indices_or_sections=4, axis=1)
......@@ -102,7 +107,6 @@ class TestCudnnLSTM(unittest.TestCase):
bias_ih_name = "_bias_ih"
weight_hh_name = "_weight_hh"
bias_hh_name = "_bias_hh"
weight_ih = param_list[weight_ih_name].numpy()
weight_ih = np.random.uniform(
-0.1, 0.1, size=weight_ih.shape).astype('float64')
......@@ -146,10 +150,9 @@ class TestCudnnLSTM(unittest.TestCase):
named_api_hidden_out = named_api_out[0]
named_api_cell_out = named_api_out[1]
np_hidden_out, np_cell_out = non_cudnn_step(
np_hidden_out, np_cell_out = cudnn_step(
step_input_np, pre_hidden_np, pre_cell_np, weight_ih, bias_ih,
weight_hh, bias_hh)
self.assertTrue(
np.allclose(
api_hidden_out.numpy(), np_hidden_out, rtol=1e-5, atol=0))
......@@ -230,7 +233,7 @@ class TestNonCudnnLSTM(unittest.TestCase):
named_api_hidden_out = named_api_out[0]
named_api_cell_out = named_api_out[1]
np_hidden_out, np_cell_out = cudnn_step(
np_hidden_out, np_cell_out = non_cudnn_step(
step_input_np, pre_hidden_np, pre_cell_np, gate_w, gate_b)
self.assertTrue(
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册