# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import numpy as np import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid import Linear from paddle.fluid.layer_helper import LayerHelper from test_imperative_base import new_program_scope import paddle.fluid.dygraph_utils as dygraph_utils from paddle.fluid.dygraph.layer_object_helper import LayerObjectHelper import paddle from paddle.fluid.framework import _in_legacy_dygraph, _test_eager_guard class MyLayer(fluid.Layer): def __init__(self): super().__init__() def forward(self, inputs): x = fluid.layers.relu(inputs) self._x_for_debug = x x = fluid.layers.elementwise_mul(x, x) x = fluid.layers.reduce_sum(x) return [x] class MLP(fluid.Layer): def __init__(self, input_size): super().__init__() self._linear1 = Linear( input_size, 3, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.1) ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.1) ), ) self._linear2 = Linear( 3, 4, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.1) ), bias_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.1) ), ) def forward(self, inputs): x = self._linear1(inputs) x = self._linear2(x) x = fluid.layers.reduce_sum(x) return x class SimpleRNNCell(fluid.Layer): def __init__(self, step_input_size, hidden_size, output_size, param_attr): super().__init__() self.step_input_size = step_input_size self.hidden_size = hidden_size self.output_size = output_size self._dtype = core.VarDesc.VarType.FP32 self.param_attr = param_attr i2h_param_shape = [self.step_input_size, self.hidden_size] h2h_param_shape = [self.hidden_size, self.hidden_size] h2o_param_shape = [self.output_size, self.hidden_size] self._i2h_w = None self._i2h_w = self.create_parameter( attr=self.param_attr, shape=i2h_param_shape, dtype=self._dtype, is_bias=False, ) self._h2h_w = self.create_parameter( attr=self.param_attr, shape=h2h_param_shape, dtype=self._dtype, is_bias=False, ) self._h2o_w = self.create_parameter( attr=self.param_attr, shape=h2o_param_shape, dtype=self._dtype, is_bias=False, ) def forward(self, input, pre_hidden): tmp_i2h = paddle.fluid.layers.nn.mul(input, self._i2h_w) tmp_h2h = paddle.fluid.layers.nn.mul(pre_hidden, self._h2h_w) hidden = paddle.add(tmp_h2h, tmp_i2h) hidden = self._helper.append_activation(hidden, act='tanh') out = paddle.fluid.layers.nn.mul(hidden, self._h2o_w) softmax_out = paddle.nn.functional.softmax(out) reduce_out = paddle.fluid.layers.nn.reduce_sum(softmax_out) return reduce_out, hidden class SimpleRNN(fluid.Layer): def __init__(self): super().__init__() self.seq_len = 4 self._cell = SimpleRNNCell( 3, 3, 3, fluid.ParamAttr(initializer=fluid.initializer.Constant(value=0.1)), ) def forward(self, inputs): outs = list() pre_hiddens = list() init_hidden = self.create_parameter( attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.1) ), shape=[1, 3], dtype='float32', is_bias=False, ) pre_hidden = init_hidden for i in range(self.seq_len): input = fluid.layers.slice( inputs, axes=[1], starts=[i], ends=[i + 1] ) input = paddle.reshape(input, shape=[1, 3]) out_softmax, pre_hidden = self._cell(input, pre_hidden) outs.append(out_softmax) return outs, pre_hiddens class TestImperative(unittest.TestCase): def functional_dygraph_context(self): self.assertFalse(fluid.dygraph.enabled()) fluid.enable_dygraph() self.assertTrue(fluid.dygraph.enabled()) np_inp = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) var_inp = paddle.to_tensor(np_inp) mlp = MLP(input_size=2) out = mlp(var_inp) dy_out1 = out.numpy() out.backward() dy_grad1 = mlp._linear1.weight.gradient() fluid.disable_dygraph() self.assertFalse(fluid.dygraph.enabled()) with fluid.dygraph.guard(): self.assertTrue(fluid.dygraph.enabled()) var_inp = paddle.to_tensor(np_inp) mlp = MLP(input_size=2) out = mlp(var_inp) dy_out2 = out.numpy() out.backward() dy_grad2 = mlp._linear1.weight.gradient() self.assertFalse(fluid.dygraph.enabled()) np.testing.assert_array_equal(dy_out1, dy_out2) np.testing.assert_array_equal(dy_grad1, dy_grad2) def test_functional_dygraph_context(self): with _test_eager_guard(): self.functional_dygraph_context() self.functional_dygraph_context() def functional_paddle_imperative_dygraph_context(self): self.assertFalse(paddle.in_dynamic_mode()) paddle.disable_static() self.assertTrue(paddle.in_dynamic_mode()) np_inp = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) var_inp = paddle.to_tensor(np_inp) mlp = MLP(input_size=2) out = mlp(var_inp) dy_out1 = out.numpy() out.backward() dy_grad1 = mlp._linear1.weight.gradient() paddle.enable_static() self.assertFalse(paddle.in_dynamic_mode()) paddle.disable_static() self.assertTrue(paddle.in_dynamic_mode()) var_inp = paddle.to_tensor(np_inp) mlp = MLP(input_size=2) out = mlp(var_inp) dy_out2 = out.numpy() out.backward() dy_grad2 = mlp._linear1.weight.gradient() paddle.enable_static() self.assertFalse(paddle.in_dynamic_mode()) np.testing.assert_array_equal(dy_out1, dy_out2) np.testing.assert_array_equal(dy_grad1, dy_grad2) def test_functional_paddle_imperative_dygraph_context(self): with _test_eager_guard(): self.functional_paddle_imperative_dygraph_context() self.functional_paddle_imperative_dygraph_context() def func_isinstance(self): var = fluid.layers.data(shape=[1], name='x', dtype='float32') self.assertTrue(isinstance(var, fluid.Variable)) with fluid.dygraph.guard(): if not _in_legacy_dygraph(): var_base = paddle.to_tensor(np.array([3, 4, 5])) self.assertTrue(isinstance(var_base, core.eager.Tensor)) else: var_base = paddle.to_tensor(np.array([3, 4, 5])) self.assertTrue(isinstance(var_base, core.VarBase)) self.assertTrue(isinstance(var_base, fluid.Variable)) def test_isinstance(self): with _test_eager_guard(): self.func_isinstance() self.func_isinstance() def func_create_varbase(self): x = np.ones([2, 2], np.float32) y = np.zeros([3, 3], np.float32) t = fluid.Tensor() t.set(x, fluid.CPUPlace()) if not _in_legacy_dygraph(): egr_tmp = fluid.core.eager.Tensor( value=x, place=fluid.core.CPUPlace() ) egr_tmp2 = fluid.core.eager.Tensor(y, fluid.core.CPUPlace()) egr_tmp3 = paddle.to_tensor(x) egr_tmp4 = fluid.core.eager.Tensor(y) egr_tmp5 = fluid.core.eager.Tensor(value=x) egr_tmp6 = fluid.core.eager.Tensor(t) np.testing.assert_array_equal(x, egr_tmp.numpy()) np.testing.assert_array_equal(y, egr_tmp2.numpy()) np.testing.assert_array_equal(x, egr_tmp3.numpy()) np.testing.assert_array_equal(y, egr_tmp4.numpy()) np.testing.assert_array_equal(x, egr_tmp5.numpy()) np.testing.assert_array_equal(x, egr_tmp6.numpy()) else: tmp = fluid.core.VarBase(value=x, place=fluid.core.CPUPlace()) tmp2 = fluid.core.VarBase(y, fluid.core.CPUPlace()) tmp3 = paddle.to_tensor(x) tmp4 = fluid.core.VarBase(y) tmp5 = fluid.core.VarBase(value=x) tmp6 = fluid.core.VarBase(t) np.testing.assert_array_equal(x, tmp.numpy()) np.testing.assert_array_equal(y, tmp2.numpy()) np.testing.assert_array_equal(x, tmp3.numpy()) np.testing.assert_array_equal(y, tmp4.numpy()) np.testing.assert_array_equal(x, tmp5.numpy()) np.testing.assert_array_equal(x, tmp6.numpy()) def test_create_varbase(self): with fluid.dygraph.guard(): with _test_eager_guard(): self.func_create_varbase() self.func_create_varbase() def test_no_grad_guard(self): data = np.array([[2, 3], [4, 5]]).astype('float32') with fluid.dygraph.guard(): l0 = fluid.Linear(2, 2) self.assertIsNone(l0.weight._grad_ivar()) l1 = fluid.Linear(2, 2) with fluid.dygraph.no_grad(): self.assertTrue(l1.weight.stop_gradient is False) tmp = l1.weight * 2 self.assertTrue(tmp.stop_gradient) x = paddle.to_tensor(data) y = paddle.add(l0(x), tmp) o = l1(y) o.backward() self.assertIsNone(tmp._grad_ivar()) self.assertIsNotNone(l0.weight._grad_ivar()) def test_paddle_imperative_no_grad_guard(self): data = np.array([[2, 3], [4, 5]]).astype('float32') with fluid.dygraph.guard(): l0 = fluid.Linear(2, 2) self.assertIsNone(l0.weight._grad_ivar()) l1 = fluid.Linear(2, 2) with paddle.no_grad(): self.assertTrue(l1.weight.stop_gradient is False) tmp = l1.weight * 2 self.assertTrue(tmp.stop_gradient) x = paddle.to_tensor(data) y = paddle.add(l0(x), tmp) o = l1(y) o.backward() self.assertIsNone(tmp._grad_ivar()) self.assertIsNotNone(l0.weight._grad_ivar()) def test_paddle_imperative_set_grad_enabled(self): data = np.array([[2, 3], [4, 5]]).astype('float32') with fluid.dygraph.guard(): l0 = fluid.Linear(2, 2) self.assertIsNone(l0.weight._grad_ivar()) l1 = fluid.Linear(2, 2) with paddle.set_grad_enabled(False): self.assertTrue(l1.weight.stop_gradient is False) tmp = l1.weight * 2 with paddle.set_grad_enabled(True): tmp2 = l1.weight * 2 self.assertTrue(tmp.stop_gradient) self.assertTrue(tmp2.stop_gradient is False) x = paddle.to_tensor(data) y = paddle.add(l0(x), tmp2) o = l1(y) o.backward() self.assertIsNone(tmp._grad_ivar()) self.assertIsNotNone(tmp2._grad_ivar()) self.assertIsNotNone(l0.weight._grad_ivar()) def test_paddle_imperative_is_grad_enabled(self): with fluid.dygraph.guard(): with paddle.set_grad_enabled(False): self.assertTrue(paddle.is_grad_enabled() is False) with paddle.set_grad_enabled(True): self.assertTrue(paddle.is_grad_enabled()) def func_sum_op(self): x = np.ones([2, 2], np.float32) with fluid.dygraph.guard(): inputs = [] for _ in range(10): tmp = paddle.to_tensor(x) tmp.stop_gradient = False inputs.append(tmp) ret = paddle.add_n(inputs) loss = fluid.layers.reduce_sum(ret) loss.backward() with fluid.dygraph.guard(): inputs2 = [] for _ in range(10): tmp = paddle.to_tensor(x) tmp.stop_gradient = False inputs2.append(tmp) ret2 = paddle.add_n(inputs2) loss2 = fluid.layers.reduce_sum(ret2) fluid.set_flags({'FLAGS_sort_sum_gradient': True}) loss2.backward() np.testing.assert_allclose(ret.numpy(), x * 10, rtol=1e-05) np.testing.assert_allclose(inputs[0].gradient(), x, rtol=1e-05) np.testing.assert_allclose(ret2.numpy(), x * 10, rtol=1e-05) a = inputs2[0].gradient() np.testing.assert_allclose(inputs2[0].gradient(), x, rtol=1e-05) def test_sum_op(self): with _test_eager_guard(): self.func_sum_op() self.func_sum_op() def func_empty_var(self): with fluid.dygraph.guard(): cur_program = fluid.Program() cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): new_variable = cur_block.create_var( name="X", shape=[-1, 23, 48], dtype='float32' ) else: new_variable = cur_block.create_var( name="X", shape=[1, 23, 48], dtype='float32' ) try: new_variable.numpy() except Exception as e: assert type(e) == ValueError try: new_variable.backward() except Exception as e: assert type(e) == core.EnforceNotMet try: new_variable.clear_gradient() except Exception as e: assert type(e) == core.EnforceNotMet def test_empty_var(self): with _test_eager_guard(): self.func_empty_var() self.func_empty_var() def func_empty_grad(self): with fluid.dygraph.guard(): x = np.ones([2, 2], np.float32) new_var = paddle.to_tensor(x) self.assertIsNone(new_var.gradient()) try: new_var.clear_gradient() except Exception as e: assert type(e) == core.EnforceNotMet with fluid.dygraph.guard(): cur_program = fluid.Program() cur_block = cur_program.current_block() # Normally, we don't allow tensor with -1 shape being created in dygraph mode, this test is not good. if _in_legacy_dygraph(): new_variable = cur_block.create_var( name="X", shape=[-1, 23, 48], dtype='float32' ) else: new_variable = cur_block.create_var( name="X", shape=[1, 23, 48], dtype='float32' ) try: new_variable.gradient() except Exception as e: assert type(e) == ValueError def test_empty_grad(self): with _test_eager_guard(): self.func_empty_grad() self.func_empty_grad() def func_set_persistable(self): with fluid.dygraph.guard(): x = np.ones([2, 2], np.float32) new_var = paddle.to_tensor(x) self.assertFalse(new_var.persistable) new_var.persistable = True self.assertTrue(new_var.persistable) def test_set_persistable(self): with _test_eager_guard(): self.func_set_persistable() self.func_set_persistable() def func_layer(self): with fluid.dygraph.guard(): l = fluid.Layer("l") self.assertRaises(NotImplementedError, l.forward, []) def test_layer(self): with _test_eager_guard(): self.func_layer() self.func_layer() def func_layer_in_out(self): np_inp = np.array([1.0, 2.0, -1.0], dtype=np.float32) with fluid.dygraph.guard(): var_inp = paddle.to_tensor(np_inp) var_inp.stop_gradient = False l = MyLayer() x = l(var_inp)[0] self.assertIsNotNone(x) dy_out = x.numpy() x.backward() dy_grad = l._x_for_debug.gradient() with fluid.dygraph.guard(): var_inp2 = paddle.to_tensor(np_inp) var_inp2.stop_gradient = False l2 = MyLayer() x2 = l2(var_inp2)[0] self.assertIsNotNone(x2) dy_out2 = x2.numpy() fluid.set_flags({'FLAGS_sort_sum_gradient': True}) x2.backward() dy_grad2 = l2._x_for_debug.gradient() with new_program_scope(): inp = fluid.layers.data( name="inp", shape=[3], append_batch_size=False ) l = MyLayer() x = l(inp)[0] param_grads = fluid.backward.append_backward( x, parameter_list=[l._x_for_debug.name] )[0] exe = fluid.Executor( fluid.CPUPlace() if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) ) static_out, static_grad = exe.run( feed={inp.name: np_inp}, fetch_list=[x.name, param_grads[1].name], ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad, static_grad) np.testing.assert_array_equal(dy_out2, static_out) np.testing.assert_array_equal(dy_grad2, static_grad) def test_layer_in_out(self): fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) with _test_eager_guard(): self.func_layer_in_out() self.func_layer_in_out() fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) def func_mlp(self): np_inp = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=np.float32) with fluid.dygraph.guard(): var_inp = paddle.to_tensor(np_inp) mlp = MLP(input_size=2) out = mlp(var_inp) dy_out = out.numpy() out.backward() dy_grad = mlp._linear1.weight.gradient() with fluid.dygraph.guard(): var_inp2 = paddle.to_tensor(np_inp) mlp2 = MLP(input_size=2) out2 = mlp2(var_inp2) dy_out2 = out2.numpy() fluid.set_flags({'FLAGS_sort_sum_gradient': True}) out2.backward() dy_grad2 = mlp2._linear1.weight.gradient() with new_program_scope(): inp = fluid.layers.data( name="inp", shape=[2, 2], append_batch_size=False ) mlp = MLP(input_size=2) out = mlp(inp) param_grads = fluid.backward.append_backward( out, parameter_list=[mlp._linear1.weight.name] )[0] exe = fluid.Executor( fluid.CPUPlace() if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) ) exe.run(fluid.default_startup_program()) static_out, static_grad = exe.run( feed={inp.name: np_inp}, fetch_list=[out.name, param_grads[1].name], ) np.testing.assert_allclose(dy_out, static_out, rtol=1e-05) np.testing.assert_allclose(dy_grad, static_grad, rtol=1e-05) np.testing.assert_allclose(dy_out2, static_out, rtol=1e-05) np.testing.assert_allclose(dy_grad2, static_grad, rtol=1e-05) params = mlp.parameters(True) self.assertEqual("linear_0.w_0", params[0].name) self.assertEqual("linear_0.b_0", params[1].name) self.assertEqual("linear_1.w_0", params[2].name) self.assertEqual("linear_1.b_0", params[3].name) self.assertEqual(len(params), 4) sublayers = mlp.sublayers() self.assertEqual(mlp._linear1, sublayers[0]) self.assertEqual(mlp._linear2, sublayers[1]) self.assertEqual(len(sublayers), 2) def test_mlp(self): with _test_eager_guard(): self.func_mlp() self.func_mlp() def test_gradient_accumulation(self): def test_single_api(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) x = paddle.to_tensor(5.0, stop_gradient=False) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_gradient() self.assertEqual(x.grad.numpy(), 0.0) for i in range(10): y = paddle.pow(x, 4.0) y.backward() self.assertEqual(x.grad.numpy(), (i + 1) * 500) x.clear_grad() self.assertEqual(x.grad.numpy(), 0.0) def test_simple_net(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) x = paddle.to_tensor(5.0, stop_gradient=False) y = paddle.to_tensor(2.0, stop_gradient=False) z = paddle.to_tensor(3.0, stop_gradient=False) def fun(x, y, z): loss1 = x * x * y loss2 = x * z loss1.backward(retain_graph=True) loss2.backward(retain_graph=True) np.testing.assert_array_equal(x.grad.numpy(), [23.0]) np.testing.assert_array_equal(y.grad.numpy(), [25.0]) np.testing.assert_array_equal(z.grad.numpy(), [5.0]) x.clear_grad() y.clear_grad() z.clear_grad() dx = paddle.grad([loss1], x, create_graph=True)[0] loss = loss1 + loss2 + dx # loss = x*x*y + x*z + 2*x*y return loss loss = fun(x, y, z) loss.backward(retain_graph=True) # x.grad = 2*x*y + z + 2*y = 27 np.testing.assert_array_equal(x.grad.numpy(), [27]) loss.backward(retain_graph=True) np.testing.assert_array_equal(x.grad.numpy(), [54]) loss.backward() np.testing.assert_array_equal(x.grad.numpy(), [81]) with self.assertRaises(RuntimeError): loss.backward() loss1 = x * x * y loss2 = x * z dx = paddle.grad([loss1], x, create_graph=True)[0] loss = loss1 + loss2 + dx loss.backward() np.testing.assert_array_equal(dx.grad.numpy(), [1]) np.testing.assert_array_equal(x.grad.numpy(), [108]) def test_mlp(sort_sum_gradient): fluid.set_flags({'FLAGS_sort_sum_gradient': sort_sum_gradient}) input_size = 5 paddle.seed(1) mlp1 = MLP(input_size=input_size) # generate the gradient of each step mlp2 = MLP(input_size=input_size) expected_weight1_grad = 0.0 expected_bias1_grad = 0.0 expected_weight2_grad = 0.0 expected_bias2_grad = 0.0 for batch_id in range(100): x = paddle.uniform([10, input_size]) detach_x = x.detach() clear_loss = mlp2(detach_x) clear_loss.backward() expected_weight1_grad = ( expected_weight1_grad + mlp2._linear1.weight.grad.numpy() ) expected_bias1_grad = ( expected_bias1_grad + mlp2._linear1.bias.grad.numpy() ) expected_weight2_grad = ( expected_weight2_grad + mlp2._linear2.weight.grad.numpy() ) expected_bias2_grad = ( expected_bias2_grad + mlp2._linear2.bias.grad.numpy() ) loss = mlp1(x) loss.backward() np.testing.assert_array_equal(loss.grad.numpy(), [1]) np.testing.assert_allclose( mlp1._linear1.weight.grad.numpy(), expected_weight1_grad, rtol=1e-05, ) np.testing.assert_allclose( mlp1._linear1.bias.grad.numpy(), expected_bias1_grad, rtol=1e-05, ) np.testing.assert_allclose( mlp1._linear2.weight.grad.numpy(), expected_weight2_grad, rtol=1e-05, ) np.testing.assert_allclose( mlp1._linear2.bias.grad.numpy(), expected_bias2_grad, rtol=1e-05, ) mlp2.clear_gradients() np.testing.assert_array_equal(clear_loss.grad.numpy(), [1]) if ((batch_id + 1) % 10) % 2 == 0: mlp1.clear_gradients() expected_weight1_grad = 0.0 expected_bias1_grad = 0.0 expected_weight2_grad = 0.0 expected_bias2_grad = 0.0 elif ((batch_id + 1) % 10) % 2 == 1: mlp1.clear_gradients() mlp1._linear1.weight._set_grad_ivar( paddle.ones([input_size, 3]) ) mlp1._linear2.weight._set_grad_ivar(paddle.ones([3, 4])) expected_weight1_grad = 1.0 expected_bias1_grad = 0.0 expected_weight2_grad = 1.0 expected_bias2_grad = 0.0 with fluid.dygraph.guard(): test_single_api(False) test_single_api(True) test_simple_net(False) test_simple_net(True) test_mlp(False) test_mlp(True) def func_dygraph_vs_static(self): np_inp1 = np.random.rand(4, 3, 3) np_inp2 = np.random.rand(4, 3, 3) # dynamic graph with fluid.dygraph.guard(): inp1 = paddle.to_tensor(np_inp1) inp2 = paddle.to_tensor(np_inp2) if np.sum(np_inp1) < np.sum(np_inp2): x = fluid.layers.elementwise_add(inp1, inp2) else: x = fluid.layers.elementwise_sub(inp1, inp2) dygraph_result = x.numpy() # static graph with new_program_scope(): inp_data1 = fluid.layers.data( name='inp1', shape=[3, 3], dtype=np.float32 ) inp_data2 = fluid.layers.data( name='inp2', shape=[3, 3], dtype=np.float32 ) a = fluid.layers.expand( paddle.reshape(fluid.layers.reduce_sum(inp_data1), [1, 1]), [4, 1], ) b = fluid.layers.expand( paddle.reshape(fluid.layers.reduce_sum(inp_data2), [1, 1]), [4, 1], ) cond = fluid.layers.less_than(x=a, y=b) ie = fluid.layers.IfElse(cond) with ie.true_block(): d1 = ie.input(inp_data1) d2 = ie.input(inp_data2) d3 = fluid.layers.elementwise_add(d1, d2) ie.output(d3) with ie.false_block(): d1 = ie.input(inp_data1) d2 = ie.input(inp_data2) d3 = fluid.layers.elementwise_sub(d1, d2) ie.output(d3) out = ie() exe = fluid.Executor( fluid.CPUPlace() if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0) ) static_result = exe.run( fluid.default_main_program(), feed={'inp1': np_inp1, 'inp2': np_inp2}, fetch_list=out, )[0] np.testing.assert_allclose(dygraph_result, static_result, rtol=1e-05) def test_dygraph_vs_static(self): with _test_eager_guard(): self.func_dygraph_vs_static() self.func_dygraph_vs_static() def func_rnn(self): np_inp = np.array( [ [1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0], [10.0, 11.0, 12.0], ] ) np_inp = np_inp.reshape((1, 4, 3)) np_inp = np_inp.astype(np.float32) with fluid.dygraph.guard(): var_inp = paddle.to_tensor(np_inp) var_inp = paddle.reshape(var_inp, shape=[1, 4, 3]) simple_rnn = SimpleRNN() outs, pre_hiddens = simple_rnn.forward(var_inp) dy_out = outs[3].numpy() outs[3].backward() dy_grad_h2o = simple_rnn._cell._h2o_w.gradient() dy_grad_h2h = simple_rnn._cell._h2h_w.gradient() dy_grad_i2h = simple_rnn._cell._i2h_w.gradient() with fluid.dygraph.guard(): var_inp2 = paddle.to_tensor(np_inp) var_inp2 = paddle.reshape(var_inp2, shape=[1, 4, 3]) simple_rnn2 = SimpleRNN() outs2, pre_hiddens2 = simple_rnn2.forward(var_inp2) dy_out2 = outs2[3].numpy() fluid.set_flags({'FLAGS_sort_sum_gradient': True}) outs2[3].backward() dy_grad_h2o2 = simple_rnn2._cell._h2o_w.gradient() dy_grad_h2h2 = simple_rnn2._cell._h2h_w.gradient() dy_grad_i2h2 = simple_rnn2._cell._i2h_w.gradient() with new_program_scope(): inp = fluid.layers.data( name="inp", shape=[1, 4, 3], append_batch_size=False ) simple_rnn = SimpleRNN() outs, pre_hiddens = simple_rnn(inp) param_grads = fluid.backward.append_backward(outs[3]) exe = fluid.Executor(fluid.CPUPlace()) exe.run(fluid.default_startup_program()) ( static_out, static_grad_h2o, static_grad_h2h, static_grad_i2h, ) = exe.run( feed={inp.name: np_inp}, fetch_list=[ outs[3].name, param_grads[0][1].name, param_grads[1][1].name, param_grads[2][1].name, ], ) np.testing.assert_array_equal(dy_out, static_out) np.testing.assert_array_equal(dy_grad_h2o, static_grad_h2o) np.testing.assert_array_equal(dy_grad_h2h, static_grad_h2h) np.testing.assert_array_equal(dy_grad_i2h, static_grad_i2h) np.testing.assert_array_equal(dy_out2, static_out) np.testing.assert_array_equal(dy_grad_h2o2, static_grad_h2o) np.testing.assert_array_equal(dy_grad_h2h2, static_grad_h2h) np.testing.assert_array_equal(dy_grad_i2h2, static_grad_i2h) def test_rnn(self): with _test_eager_guard(): self.func_rnn() self.func_rnn() def func_layer_attrs(self): layer = fluid.dygraph.Layer("test") layer.test_attr = 1 self.assertFalse(hasattr(layer, "whatever")) self.assertTrue(hasattr(layer, "test_attr")) self.assertEqual(layer.test_attr, 1) my_layer = MyLayer() my_layer.w1 = my_layer.create_parameter([3, 3]) my_layer.add_parameter('w2', None) self.assertEqual(len(my_layer.parameters()), 1) self.assertRaises(TypeError, my_layer.__setattr__, 'w1', 'str') my_layer.w1 = None self.assertEqual(len(my_layer.parameters()), 0) my_layer.l1 = fluid.dygraph.Linear(3, 3) self.assertEqual(len(my_layer.sublayers()), 1) self.assertRaises(TypeError, my_layer.__setattr__, 'l1', 'str') my_layer.l1 = None self.assertEqual(len(my_layer.sublayers()), 0) def test_layer_attrs(self): with _test_eager_guard(): self.func_layer_attrs() self.func_layer_attrs() class TestDygraphUtils(unittest.TestCase): def func_append_activation_in_dygraph_exception(self): with new_program_scope(): np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32) a = fluid.layers.data("a", [10, 20]) func = dygraph_utils._append_activation_in_dygraph self.assertRaises(AssertionError, func, a, act="sigmoid") def test_append_activation_in_dygraph_exception(self): with _test_eager_guard(): self.func_append_activation_in_dygraph_exception() self.func_append_activation_in_dygraph_exception() def func_append_activation_in_dygraph1(self): a_np = np.random.random(size=(10, 20, 30)).astype(np.float32) func = dygraph_utils._append_activation_in_dygraph with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) res1 = func(a, act="hard_sigmoid") res2 = fluid.layers.hard_sigmoid(a) np.testing.assert_array_equal(res1.numpy(), res2.numpy()) def test_append_activation_in_dygraph1(self): with _test_eager_guard(): self.func_append_activation_in_dygraph1() self.func_append_activation_in_dygraph1() def func_append_activation_in_dygraph2(self): a_np = np.random.random(size=(10, 20, 30)).astype(np.float32) func = dygraph_utils._append_activation_in_dygraph with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) res1 = func(a, act="sigmoid", use_mkldnn=True, use_cudnn=True) res2 = paddle.nn.functional.sigmoid(a) np.testing.assert_allclose(res1.numpy(), res2.numpy(), rtol=1e-05) def test_append_activation_in_dygraph2(self): with _test_eager_guard(): self.func_append_activation_in_dygraph2() self.func_append_activation_in_dygraph2() def func_append_activation_in_dygraph3(self): a_np = np.random.random(size=(10, 20, 30)).astype(np.float32) helper = LayerObjectHelper(fluid.unique_name.generate("test")) func = helper.append_activation with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) res1 = func(a, act="sigmoid", use_cudnn=True) res2 = paddle.nn.functional.sigmoid(a) np.testing.assert_array_equal(res1.numpy(), res2.numpy()) def test_append_activation_in_dygraph3(self): with _test_eager_guard(): self.func_append_activation_in_dygraph3() self.func_append_activation_in_dygraph3() def func_append_activation_in_dygraph_use_mkldnn(self): a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) helper = LayerHelper( fluid.unique_name.generate("test"), act="relu", use_mkldnn=True ) func = helper.append_activation with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) res1 = func(a) res2 = fluid.layers.relu(a) np.testing.assert_array_equal(res1.numpy(), res2.numpy()) def test_append_activation_in_dygraph_use_mkldnn(self): with _test_eager_guard(): self.func_append_activation_in_dygraph_use_mkldnn() self.func_append_activation_in_dygraph_use_mkldnn() def func_append_activation_in_dygraph_global_use_mkldnn(self): a_np = np.random.uniform(-2, 2, (10, 20, 30)).astype(np.float32) helper = LayerHelper(fluid.unique_name.generate("test"), act="relu") func = helper.append_activation with fluid.dygraph.guard(fluid.core.CPUPlace()): a = paddle.to_tensor(a_np) fluid.set_flags({'FLAGS_use_mkldnn': True}) try: res1 = func(a) finally: fluid.set_flags({'FLAGS_use_mkldnn': False}) res2 = fluid.layers.relu(a) np.testing.assert_array_equal(res1.numpy(), res2.numpy()) def test_append_activation_in_dygraph_global_use_mkldnn(self): with _test_eager_guard(): self.func_append_activation_in_dygraph_global_use_mkldnn() self.func_append_activation_in_dygraph_global_use_mkldnn() def func_append_bias_in_dygraph_exception(self): with new_program_scope(): np_inp = np.random.random(size=(10, 20, 30)).astype(np.float32) a = fluid.layers.data("a", [10, 20]) func = dygraph_utils._append_bias_in_dygraph self.assertRaises(AssertionError, func, a) def test_append_bias_in_dygraph_exception(self): with _test_eager_guard(): self.func_append_bias_in_dygraph_exception() self.func_append_bias_in_dygraph_exception() def func_append_bias_in_dygraph(self): a_np = np.random.random(size=(10, 20, 30)).astype(np.float32) func = dygraph_utils._append_bias_in_dygraph with fluid.dygraph.guard(): a = paddle.to_tensor(a_np) res1 = func(a, bias=a) res2 = paddle.add(a, a) np.testing.assert_array_equal(res1.numpy(), res2.numpy()) def test_append_bias_in_dygraph(self): with _test_eager_guard(): self.func_append_bias_in_dygraph() self.func_append_bias_in_dygraph() class TestDygraphGuardWithError(unittest.TestCase): def func_without_guard(self): with fluid.dygraph.guard(): x = paddle.to_tensor(np.zeros([10, 10])) with self.assertRaisesRegexp( TypeError, "Please use `with fluid.dygraph.guard()" ): y = fluid.layers.matmul(x, x) def test_without_guard(self): with _test_eager_guard(): self.func_without_guard() self.func_without_guard() class TestMetaclass(unittest.TestCase): def func_metaclass(self): self.assertEqual(type(MyLayer).__name__, 'type') self.assertNotEqual(type(MyLayer).__name__, 'pybind11_type') if not _in_legacy_dygraph(): self.assertEqual( type(paddle.fluid.core.eager.Tensor).__name__, 'type' ) else: self.assertEqual( type(paddle.fluid.core.VarBase).__name__, 'pybind11_type' ) def test_metaclass(self): with _test_eager_guard(): self.func_metaclass() self.func_metaclass() if __name__ == '__main__': paddle.enable_static() unittest.main()