test_understand_sentiment.py 11.6 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14
from __future__ import print_function
D
dzhwinter 已提交
15

16
import unittest
17
import paddle.fluid as fluid
18 19
import paddle.v2 as paddle
import contextlib
20
import math
21
import numpy as np
22
import sys
23 24 25 26


def convolution_net(data, label, input_dim, class_dim=2, emb_dim=32,
                    hid_dim=32):
27 28
    emb = fluid.layers.embedding(
        input=data, size=[input_dim, emb_dim], is_sparse=True)
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
    conv_3 = fluid.nets.sequence_conv_pool(
        input=emb,
        num_filters=hid_dim,
        filter_size=3,
        act="tanh",
        pool_type="sqrt")
    conv_4 = fluid.nets.sequence_conv_pool(
        input=emb,
        num_filters=hid_dim,
        filter_size=4,
        act="tanh",
        pool_type="sqrt")
    prediction = fluid.layers.fc(input=[conv_3, conv_4],
                                 size=class_dim,
                                 act="softmax")
    cost = fluid.layers.cross_entropy(input=prediction, label=label)
    avg_cost = fluid.layers.mean(x=cost)
    accuracy = fluid.layers.accuracy(input=prediction, label=label)
47
    return avg_cost, accuracy, prediction
Q
QI JUN 已提交
48 49


Y
Yu Yang 已提交
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
def dyn_rnn_lstm(data, label, input_dim, class_dim=2, emb_dim=32,
                 lstm_size=128):
    emb = fluid.layers.embedding(
        input=data, size=[input_dim, emb_dim], is_sparse=True)
    sentence = fluid.layers.fc(input=emb, size=lstm_size, act='tanh')

    rnn = fluid.layers.DynamicRNN()
    with rnn.block():
        word = rnn.step_input(sentence)
        prev_hidden = rnn.memory(value=0.0, shape=[lstm_size])
        prev_cell = rnn.memory(value=0.0, shape=[lstm_size])

        def gate_common(ipt, hidden, size):
            gate0 = fluid.layers.fc(input=ipt, size=size, bias_attr=True)
            gate1 = fluid.layers.fc(input=hidden, size=size, bias_attr=False)
            return gate0 + gate1

        forget_gate = fluid.layers.sigmoid(x=gate_common(word, prev_hidden,
                                                         lstm_size))
        input_gate = fluid.layers.sigmoid(x=gate_common(word, prev_hidden,
                                                        lstm_size))
        output_gate = fluid.layers.sigmoid(x=gate_common(word, prev_hidden,
                                                         lstm_size))
        cell_gate = fluid.layers.sigmoid(x=gate_common(word, prev_hidden,
                                                       lstm_size))

        cell = forget_gate * prev_cell + input_gate * cell_gate
        hidden = output_gate * fluid.layers.tanh(x=cell)
        rnn.update_memory(prev_cell, cell)
        rnn.update_memory(prev_hidden, hidden)
        rnn.output(hidden)

    last = fluid.layers.sequence_last_step(rnn())
    prediction = fluid.layers.fc(input=last, size=class_dim, act="softmax")
    cost = fluid.layers.cross_entropy(input=prediction, label=label)
    avg_cost = fluid.layers.mean(x=cost)
    accuracy = fluid.layers.accuracy(input=prediction, label=label)
    return avg_cost, accuracy, prediction


Y
Yu Yang 已提交
90 91 92
def stacked_lstm_net(data,
                     label,
                     input_dim,
Q
QI JUN 已提交
93 94 95 96 97 98
                     class_dim=2,
                     emb_dim=128,
                     hid_dim=512,
                     stacked_num=3):
    assert stacked_num % 2 == 1

99 100
    emb = fluid.layers.embedding(
        input=data, size=[input_dim, emb_dim], is_sparse=True)
Q
QI JUN 已提交
101 102 103
    # add bias attr

    # TODO(qijun) linear act
104 105
    fc1 = fluid.layers.fc(input=emb, size=hid_dim)
    lstm1, cell1 = fluid.layers.dynamic_lstm(input=fc1, size=hid_dim)
Q
QI JUN 已提交
106 107 108 109

    inputs = [fc1, lstm1]

    for i in range(2, stacked_num + 1):
110 111
        fc = fluid.layers.fc(input=inputs, size=hid_dim)
        lstm, cell = fluid.layers.dynamic_lstm(
Q
QI JUN 已提交
112 113 114
            input=fc, size=hid_dim, is_reverse=(i % 2) == 0)
        inputs = [fc, lstm]

115 116
    fc_last = fluid.layers.sequence_pool(input=inputs[0], pool_type='max')
    lstm_last = fluid.layers.sequence_pool(input=inputs[1], pool_type='max')
Q
QI JUN 已提交
117

118 119 120 121 122
    prediction = fluid.layers.fc(input=[fc_last, lstm_last],
                                 size=class_dim,
                                 act='softmax')
    cost = fluid.layers.cross_entropy(input=prediction, label=label)
    avg_cost = fluid.layers.mean(x=cost)
123
    accuracy = fluid.layers.accuracy(input=prediction, label=label)
124
    return avg_cost, accuracy, prediction
Q
QI JUN 已提交
125

126

127 128 129 130 131 132
def create_random_lodtensor(lod, place, low, high):
    data = np.random.random_integers(low, high, [lod[-1], 1]).astype("int64")
    res = fluid.LoDTensor()
    res.set(data, place)
    res.set_lod([lod])
    return res
133

134

135
def train(word_dict, net_method, use_cuda, parallel=False, save_dirname=None):
136 137
    BATCH_SIZE = 128
    PASS_NUM = 5
Q
QI JUN 已提交
138 139 140
    dict_dim = len(word_dict)
    class_dim = 2

Y
Yu Yang 已提交
141 142 143
    data = fluid.layers.data(
        name="words", shape=[1], dtype="int64", lod_level=1)
    label = fluid.layers.data(name="label", shape=[1], dtype="int64")
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167

    if not parallel:
        cost, acc_out, prediction = net_method(
            data, label, input_dim=dict_dim, class_dim=class_dim)
    else:
        places = fluid.layers.get_places()
        pd = fluid.layers.ParallelDo(places)
        with pd.do():
            cost, acc, _ = net_method(
                pd.read_input(data),
                pd.read_input(label),
                input_dim=dict_dim,
                class_dim=class_dim)
            pd.write_output(cost)
            pd.write_output(acc)

        cost, acc = pd()
        cost = fluid.layers.mean(x=cost)
        acc_out = fluid.layers.mean(x=acc)
        prediction = None
        assert save_dirname is None

    adagrad = fluid.optimizer.Adagrad(learning_rate=0.002)
    adagrad.minimize(cost)
Q
QI JUN 已提交
168 169 170 171 172

    train_data = paddle.batch(
        paddle.reader.shuffle(
            paddle.dataset.imdb.train(word_dict), buf_size=1000),
        batch_size=BATCH_SIZE)
173
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
174
    exe = fluid.Executor(place)
Y
Yu Yang 已提交
175
    feeder = fluid.DataFeeder(feed_list=[data, label], place=place)
Q
QI JUN 已提交
176

177
    exe.run(fluid.default_startup_program())
Q
QI JUN 已提交
178 179 180

    for pass_id in xrange(PASS_NUM):
        for data in train_data():
Y
Yu Yang 已提交
181 182 183
            cost_val, acc_val = exe.run(fluid.default_main_program(),
                                        feed=feeder.feed(data),
                                        fetch_list=[cost, acc_out])
184 185
            print("cost=" + str(cost_val) + " acc=" + str(acc_val))
            if cost_val < 0.4 and acc_val > 0.8:
186 187 188
                if save_dirname is not None:
                    fluid.io.save_inference_model(save_dirname, ["words"],
                                                  prediction, exe)
189
                return
190 191
            if math.isnan(float(cost_val)):
                sys.exit("got NaN loss, training failed.")
192 193 194 195
    raise AssertionError("Cost is too large for {0}".format(
        net_method.__name__))


196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
def infer(use_cuda, save_dirname=None):
    if save_dirname is None:
        return

    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
    exe = fluid.Executor(place)

    # Use fluid.io.load_inference_model to obtain the inference program desc,
    # the feed_target_names (the names of variables that will be feeded 
    # data using feed operators), and the fetch_targets (variables that 
    # we want to obtain data from using fetch operators).
    [inference_program, feed_target_names,
     fetch_targets] = fluid.io.load_inference_model(save_dirname, exe)

    lod = [0, 4, 10]
    word_dict = paddle.dataset.imdb.word_dict()
    tensor_words = create_random_lodtensor(
        lod, place, low=0, high=len(word_dict) - 1)

    # Construct feed as a dictionary of {feed_target_name: feed_target_data}
    # and results will contain a list of data corresponding to fetch_targets.
    assert feed_target_names[0] == "words"
    results = exe.run(inference_program,
                      feed={feed_target_names[0]: tensor_words},
                      fetch_list=fetch_targets,
                      return_numpy=False)
    print(results[0].lod())
    np_data = np.array(results[0])
    print("Inference Shape: ", np_data.shape)
    print("Inference results: ", np_data)


228
def main(word_dict, net_method, use_cuda, parallel=False, save_dirname=None):
229 230 231
    if use_cuda and not fluid.core.is_compiled_with_cuda():
        return

232 233 234 235 236 237
    train(
        word_dict,
        net_method,
        use_cuda,
        parallel=parallel,
        save_dirname=save_dirname)
238 239 240
    infer(use_cuda, save_dirname)


241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
class TestUnderstandSentiment(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.word_dict = paddle.dataset.imdb.word_dict()

    @contextlib.contextmanager
    def new_program_scope(self):
        prog = fluid.Program()
        startup_prog = fluid.Program()
        scope = fluid.core.Scope()
        with fluid.scope_guard(scope):
            with fluid.program_guard(prog, startup_prog):
                yield

    def test_conv_cpu(self):
        with self.new_program_scope():
257 258 259 260 261
            main(
                self.word_dict,
                net_method=convolution_net,
                use_cuda=False,
                save_dirname="understand_sentiment.inference.model")
262

263 264 265 266 267 268 269 270 271
    def test_conv_cpu_parallel(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=convolution_net,
                use_cuda=False,
                parallel=True)

    @unittest.skip(reason="make CI faster")
272 273 274 275
    def test_stacked_lstm_cpu(self):
        with self.new_program_scope():
            main(self.word_dict, net_method=stacked_lstm_net, use_cuda=False)

276 277 278 279 280 281 282 283
    def test_stacked_lstm_cpu_parallel(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=stacked_lstm_net,
                use_cuda=False,
                parallel=True)

284 285
    def test_conv_gpu(self):
        with self.new_program_scope():
286 287 288 289 290 291 292 293 294 295 296 297 298
            main(
                self.word_dict,
                net_method=convolution_net,
                use_cuda=True,
                save_dirname="understand_sentiment.inference.model")

    def test_conv_gpu_parallel(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=convolution_net,
                use_cuda=True,
                parallel=True)
299

300
    @unittest.skip(reason="make CI faster")
301 302 303
    def test_stacked_lstm_gpu(self):
        with self.new_program_scope():
            main(self.word_dict, net_method=stacked_lstm_net, use_cuda=True)
Q
QI JUN 已提交
304

305 306 307 308 309 310 311 312
    def test_stacked_lstm_gpu_parallel(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=stacked_lstm_net,
                use_cuda=True,
                parallel=True)

Y
Yu Yang 已提交
313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
    @unittest.skip(reason='make CI faster')
    def test_dynrnn_lstm_gpu(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=dyn_rnn_lstm,
                use_cuda=True,
                parallel=False)

    def test_dynrnn_lstm_gpu_parallel(self):
        with self.new_program_scope():
            main(
                self.word_dict,
                net_method=dyn_rnn_lstm,
                use_cuda=True,
                parallel=True)

Q
QI JUN 已提交
330 331

if __name__ == '__main__':
332
    unittest.main()