test_multihead_attention.py

#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import paddle.v2.fluid as fluid
import paddle.v2.fluid.core as core
import numpy as np

import pdb


class TestMultiheadAttention(unittest.TestCase):
    def gen_random_input(self):
        """Generate random input data.
        """
        # batch_size, max_sequence_length, hidden dimension
        self.input_shape = (3, 13, 16)
        self.queries = np.random.random(size=self.input_shape).astype("float32")
        self.keys = np.random.random(size=self.input_shape).astype("float32")

    def set_program(self):
        """Build the test program.
        """
        queries = fluid.layers.data(
            name="queries",
            shape=self.input_shape,
            dtype="float32",
            append_batch_size=False)
        queries.stop_gradient = False
        keys = fluid.layers.data(
            name="keys",
            shape=self.input_shape,
            dtype="float32",
            append_batch_size=False)
        keys.stop_gradient = False

        contexts, att_scores = fluid.nets.scaled_dot_product_attention(
            queries=queries,
            keys=keys,
            values=keys,
            num_heads=8,
            dropout_rate=0.)
        out = fluid.layers.reduce_sum(contexts, dim=None)
        fluid.backward.append_backward(loss=out)

        self.fetch_list = [contexts]

    def run_program(self):
        """Run the test program.
        """
        places = [core.CPUPlace()]
        if core.is_compile_gpu():
            places.append(core.CUDAPlace(0))

        for place in places:
            self.set_inputs(place)
            exe = fluid.Executor(place)

            output = exe.run(fluid.default_main_program(),
                             feed=self.inputs,
                             fetch_list=self.fetch_list,
                             return_numpy=True)
            self.op_output = output

    def set_inputs(self, place):
        """Set the randomly generated data to the test program.
        """
        self.inputs = {}
        queries = fluid.Tensor()
        queries.set(self.queries, place)

        keys = fluid.Tensor()
        keys.set(self.keys, place)

        self.inputs["keys"] = keys
        self.inputs["values"] = values

    def test_multihead_attention(self):
        self.gen_random_input()

        self.set_program()
        pdb.set_trace()
        self.run_program()

        expect_output = self.l2_normalize(self.data, axis, epsilon)

        # check output
        self.assertTrue(np.allclose(self.op_output, expect_output, atol=0.001))


if __name__ == '__main__':
    unittest.main()