test_weight_decay_extend.py

#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
from functools import partial
import numpy as np
import paddle
import paddle.fluid as fluid
import contextlib

paddle.enable_static()

SEED = 2020


def fake_imdb_reader(
    word_dict_size,
    sample_num,
    lower_seq_len=100,
    upper_seq_len=200,
    class_dim=2,
):
    def __reader__():
        for _ in range(sample_num):
            length = np.random.random_integers(
                low=lower_seq_len, high=upper_seq_len, size=[1]
            )[0]
            ids = np.random.random_integers(
                low=0, high=word_dict_size - 1, size=[length]
            ).astype('int64')
            label = np.random.random_integers(
                low=0, high=class_dim - 1, size=[1]
            ).astype('int64')[0]
            yield ids, label

    return __reader__


def get_places():
    places = [fluid.CPUPlace()]
    if fluid.core.is_compiled_with_cuda():
        places.append(fluid.CUDAPlace(0))
    return places


@contextlib.contextmanager
def prog_scope_guard(main_prog, startup_prog):
    scope = fluid.core.Scope()
    with fluid.unique_name.guard():
        with fluid.scope_guard(scope):
            with fluid.program_guard(main_prog, startup_prog):
                yield


def bow_net(
    data,
    label,
    dict_dim,
    is_sparse=False,
    emb_dim=128,
    hid_dim=128,
    hid_dim2=96,
    class_dim=2,
):
    """
    BOW net
    This model is from https://github.com/PaddlePaddle/models:
    fluid/PaddleNLP/text_classification/nets.py
    """
    emb = fluid.layers.embedding(
        input=data, is_sparse=is_sparse, size=[dict_dim, emb_dim]
    )
    bow = paddle.static.nn.sequence_lod.sequence_pool(
        input=emb, pool_type='sum'
    )
    bow_tanh = paddle.tanh(bow)
    fc_1 = paddle.static.nn.fc(x=bow_tanh, size=hid_dim, activation="tanh")
    fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim2, activation="tanh")
    prediction = paddle.static.nn.fc(
        x=[fc_2], size=class_dim, activation="softmax"
    )
    cost = paddle.nn.functional.cross_entropy(
        input=prediction, label=label, reduction='none', use_softmax=False
    )
    avg_cost = paddle.mean(x=cost)

    return avg_cost


class TestWeightDecay(unittest.TestCase):
    def setUp(self):
        # set seed
        np.random.seed(SEED)
        paddle.seed(SEED)
        paddle.framework.random._manual_program_seed(SEED)
        # configs
        self.word_dict_len = 5147
        batch_size = 2
        reader = fake_imdb_reader(self.word_dict_len, batch_size * 100)
        reader = paddle.batch(reader, batch_size=batch_size)()
        self.train_data = [next(reader) for _ in range(3)]
        self.learning_rate = 0.5

    def run_program(self, place, feed_list):
        exe = fluid.Executor(place)
        feeder = fluid.DataFeeder(feed_list=feed_list, place=place)
        exe.run(fluid.default_startup_program())

        main_prog = fluid.default_main_program()
        param_list = [var.name for var in main_prog.block(0).all_parameters()]

        param_sum = []
        for data in self.train_data:
            out = exe.run(
                main_prog, feed=feeder.feed(data), fetch_list=param_list
            )
            p_sum = 0
            for v in out:
                p_sum += np.sum(np.abs(v))
            param_sum.append(p_sum)
        return param_sum

    def check_weight_decay(self, place, model):
        main_prog = fluid.framework.Program()
        startup_prog = fluid.framework.Program()

        with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog):
            data = paddle.static.data(
                name="words", shape=[-1, 1], dtype="int64", lod_level=1
            )
            label = paddle.static.data(
                name="label", shape=[-1, 1], dtype="int64"
            )
            avg_cost = model(data, label, self.word_dict_len)
            AdamW = fluid.contrib.extend_with_decoupled_weight_decay(
                fluid.optimizer.Adam
            )

            optimizer = AdamW(
                learning_rate=self.learning_rate,
                weight_decay=self.learning_rate,
            )

            optimizer.minimize(avg_cost)
            param_sum = self.run_program(place, [data, label])

        return param_sum

    def check_weight_decay2(self, place, model):
        main_prog = fluid.framework.Program()
        startup_prog = fluid.framework.Program()

        with prog_scope_guard(main_prog=main_prog, startup_prog=startup_prog):
            data = paddle.static.data(
                name="words", shape=[-1, 1], dtype="int64", lod_level=1
            )
            label = paddle.static.data(
                name="label", shape=[-1, 1], dtype="int64"
            )

            avg_cost = model(data, label, self.word_dict_len)

            optimizer = fluid.optimizer.Adam(learning_rate=self.learning_rate)

            params_grads = optimizer.backward(avg_cost)

            param_list = [
                (var, var * self.learning_rate)
                for var in main_prog.block(0).all_parameters()
            ]

            for params in param_list:
                updated_p = paddle.subtract(x=params[0], y=params[1])
                fluid.layers.assign(input=updated_p, output=params[0])

            optimizer.apply_optimize(avg_cost, startup_prog, params_grads)

            param_sum = self.run_program(place, [data, label])
        return param_sum

    def test_weight_decay(self):
        for place in get_places():
            model = partial(bow_net, is_sparse=False)
            param_sum1 = self.check_weight_decay(place, model)
            param_sum2 = self.check_weight_decay2(place, model)

            for i in range(len(param_sum1)):
                np.testing.assert_allclose(
                    param_sum1[i],
                    param_sum2[i],
                    rtol=1e-05,
                    err_msg='Current place: {}, i: {}, sum1: {}, sum2: {}'.format(
                        place,
                        i,
                        param_sum1[i][
                            ~np.isclose(param_sum1[i], param_sum2[i])
                        ],
                        param_sum2[i][
                            ~np.isclose(param_sum1[i], param_sum2[i])
                        ],
                    ),
                )


if __name__ == '__main__':
    unittest.main()