test_label_semantic_roles.py 11.4 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Q
Qiao Longfei 已提交
15 16
import math

Q
Qiao Longfei 已提交
17 18 19
import numpy as np
import paddle.v2 as paddle
import paddle.v2.dataset.conll05 as conll05
20
import paddle.v2.fluid as fluid
21
from paddle.v2.fluid.initializer import init_on_cpu
22
import contextlib
23
import time
24
import unittest
Q
Qiao Longfei 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39

word_dict, verb_dict, label_dict = conll05.get_dict()
word_dict_len = len(word_dict)
label_dict_len = len(label_dict)
pred_len = len(verb_dict)

mark_dict_len = 2
word_dim = 32
mark_dim = 5
hidden_dim = 512
depth = 8
mix_hidden_lr = 1e-3

IS_SPARSE = True
PASS_NUM = 10
40
BATCH_SIZE = 10
Q
Qiao Longfei 已提交
41 42 43 44 45 46 47 48 49 50

embedding_name = 'emb'


def load_parameter(file_name, h, w):
    with open(file_name, 'rb') as f:
        f.read(16)  # skip header.
        return np.fromfile(f, dtype=np.float32).reshape(h, w)


Y
Yu Yang 已提交
51 52
def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark,
            **ignored):
Q
Qiao Longfei 已提交
53
    # 8 features
54
    predicate_embedding = fluid.layers.embedding(
Q
Qiao Longfei 已提交
55 56
        input=predicate,
        size=[pred_len, word_dim],
F
fengjiayi 已提交
57
        dtype='float32',
Q
Qiao Longfei 已提交
58
        is_sparse=IS_SPARSE,
Y
Yu Yang 已提交
59
        param_attr='vemb')
Q
Qiao Longfei 已提交
60

61
    mark_embedding = fluid.layers.embedding(
Q
Qiao Longfei 已提交
62 63
        input=mark,
        size=[mark_dict_len, mark_dim],
F
fengjiayi 已提交
64
        dtype='float32',
Q
Qiao Longfei 已提交
65 66 67 68
        is_sparse=IS_SPARSE)

    word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2]
    emb_layers = [
69
        fluid.layers.embedding(
Q
Qiao Longfei 已提交
70 71
            size=[word_dict_len, word_dim],
            input=x,
Y
Yu Yang 已提交
72 73
            param_attr=fluid.ParamAttr(
                name=embedding_name, trainable=False)) for x in word_input
Q
Qiao Longfei 已提交
74 75 76 77 78
    ]
    emb_layers.append(predicate_embedding)
    emb_layers.append(mark_embedding)

    hidden_0_layers = [
79
        fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers
Q
Qiao Longfei 已提交
80 81
    ]

82
    hidden_0 = fluid.layers.sums(input=hidden_0_layers)
Q
Qiao Longfei 已提交
83

84
    lstm_0 = fluid.layers.dynamic_lstm(
Q
Qiao Longfei 已提交
85 86 87 88 89 90 91 92 93 94
        input=hidden_0,
        size=hidden_dim,
        candidate_activation='relu',
        gate_activation='sigmoid',
        cell_activation='sigmoid')

    # stack L-LSTM and R-LSTM with direct edges
    input_tmp = [hidden_0, lstm_0]

    for i in range(1, depth):
95 96 97
        mix_hidden = fluid.layers.sums(input=[
            fluid.layers.fc(input=input_tmp[0], size=hidden_dim),
            fluid.layers.fc(input=input_tmp[1], size=hidden_dim)
Q
Qiao Longfei 已提交
98 99
        ])

100
        lstm = fluid.layers.dynamic_lstm(
Q
Qiao Longfei 已提交
101 102 103 104 105 106 107 108 109
            input=mix_hidden,
            size=hidden_dim,
            candidate_activation='relu',
            gate_activation='sigmoid',
            cell_activation='sigmoid',
            is_reverse=((i % 2) == 1))

        input_tmp = [mix_hidden, lstm]

110 111 112
    feature_out = fluid.layers.sums(input=[
        fluid.layers.fc(input=input_tmp[0], size=label_dict_len),
        fluid.layers.fc(input=input_tmp[1], size=label_dict_len)
Q
Qiao Longfei 已提交
113 114 115 116 117 118 119 120 121 122 123 124 125 126
    ])

    return feature_out


def to_lodtensor(data, place):
    seq_lens = [len(seq) for seq in data]
    cur_len = 0
    lod = [cur_len]
    for l in seq_lens:
        cur_len += l
        lod.append(cur_len)
    flattened_data = np.concatenate(data, axis=0).astype("int64")
    flattened_data = flattened_data.reshape([len(flattened_data), 1])
127
    res = fluid.LoDTensor()
Q
Qiao Longfei 已提交
128 129 130 131 132
    res.set(flattened_data, place)
    res.set_lod([lod])
    return res


133 134 135 136 137 138 139 140 141
def create_random_lodtensor(lod, place, low, high):
    data = np.random.random_integers(low, high, [lod[-1], 1]).astype("int64")
    res = fluid.LoDTensor()
    res.set(data, place)
    res.set_lod([lod])
    return res


def train(use_cuda, save_dirname=None):
Q
Qiao Longfei 已提交
142
    # define network topology
Y
Yu Yang 已提交
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
    word = fluid.layers.data(
        name='word_data', shape=[1], dtype='int64', lod_level=1)
    predicate = fluid.layers.data(
        name='verb_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n2 = fluid.layers.data(
        name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n1 = fluid.layers.data(
        name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_0 = fluid.layers.data(
        name='ctx_0_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p1 = fluid.layers.data(
        name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p2 = fluid.layers.data(
        name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1)
    mark = fluid.layers.data(
        name='mark_data', shape=[1], dtype='int64', lod_level=1)
    feature_out = db_lstm(**locals())
    target = fluid.layers.data(
        name='target', shape=[1], dtype='int64', lod_level=1)
162
    crf_cost = fluid.layers.linear_chain_crf(
Q
Qiao Longfei 已提交
163 164
        input=feature_out,
        label=target,
Y
Yu Yang 已提交
165 166
        param_attr=fluid.ParamAttr(
            name='crfw', learning_rate=mix_hidden_lr))
167
    avg_cost = fluid.layers.mean(x=crf_cost)
Q
Qiao Longfei 已提交
168

Q
Qiao Longfei 已提交
169
    # TODO(qiao)
Q
Qiao Longfei 已提交
170
    # check other optimizers and check why out will be NAN
171 172 173 174 175 176 177 178 179 180
    global_step = fluid.layers.create_global_var(
        shape=[1], value=0, dtype='float32', force_cpu=True, persistable=True)
    sgd_optimizer = fluid.optimizer.SGD(
        learning_rate=fluid.learning_rate_decay.exponential_decay(
            learning_rate=0.0001,
            global_step=global_step,
            decay_steps=100000,
            decay_rate=0.5,
            staircase=True),
        global_step=global_step)
181
    sgd_optimizer.minimize(avg_cost)
Q
Qiao Longfei 已提交
182

Q
Qiao Longfei 已提交
183 184 185
    # TODO(qiao)
    # add dependency track and move this config before optimizer
    crf_decode = fluid.layers.crf_decoding(
Q
Qiao Longfei 已提交
186 187
        input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))

G
guosheng 已提交
188
    chunk_evaluator = fluid.evaluator.ChunkEvaluator(
Q
Qiao Longfei 已提交
189
        input=crf_decode,
Q
Qiao Longfei 已提交
190
        label=target,
Q
Qiao Longfei 已提交
191 192
        chunk_scheme="IOB",
        num_chunk_types=int(math.ceil((label_dict_len - 1) / 2.0)))
Q
Qiao Longfei 已提交
193

Q
Qiao Longfei 已提交
194 195 196 197
    train_data = paddle.batch(
        paddle.reader.shuffle(
            paddle.dataset.conll05.test(), buf_size=8192),
        batch_size=BATCH_SIZE)
198 199

    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
Y
Yu Yang 已提交
200 201 202 203 204
    feeder = fluid.DataFeeder(
        feed_list=[
            word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target
        ],
        place=place)
205
    exe = fluid.Executor(place)
Q
Qiao Longfei 已提交
206

207
    exe.run(fluid.default_startup_program())
Q
Qiao Longfei 已提交
208

Y
Yang Yu 已提交
209
    embedding_param = fluid.global_scope().find_var(embedding_name).get_tensor()
Q
Qiao Longfei 已提交
210 211 212
    embedding_param.set(
        load_parameter(conll05.get_embedding(), word_dict_len, word_dim), place)

213
    start_time = time.time()
Q
Qiao Longfei 已提交
214 215
    batch_id = 0
    for pass_id in xrange(PASS_NUM):
G
guosheng 已提交
216
        chunk_evaluator.reset(exe)
Q
Qiao Longfei 已提交
217
        for data in train_data():
218 219 220 221 222 223
            cost, precision, recall, f1_score = exe.run(
                fluid.default_main_program(),
                feed=feeder.feed(data),
                fetch_list=[avg_cost] + chunk_evaluator.metrics)
            pass_precision, pass_recall, pass_f1_score = chunk_evaluator.eval(
                exe)
Q
Qiao Longfei 已提交
224

Q
Qiao Longfei 已提交
225
            if batch_id % 10 == 0:
226 227 228 229 230
                print("avg_cost:" + str(cost) + " precision:" + str(
                    precision) + " recall:" + str(recall) + " f1_score:" + str(
                        f1_score) + " pass_precision:" + str(
                            pass_precision) + " pass_recall:" + str(pass_recall)
                      + " pass_f1_score:" + str(pass_f1_score))
231 232 233
                if batch_id != 0:
                    print("second per batch: " + str((time.time() - start_time)
                                                     / batch_id))
234 235 236 237 238 239 240 241 242
                # Set the threshold low to speed up the CI test
                if float(pass_precision) > 0.05:
                    if save_dirname is not None:
                        fluid.io.save_inference_model(save_dirname, [
                            'word_data', 'verb_data', 'ctx_n2_data',
                            'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data',
                            'ctx_p2_data', 'mark_data'
                        ], [feature_out], exe)
                    return
Q
Qiao Longfei 已提交
243 244 245 246

            batch_id = batch_id + 1


247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
def infer(use_cuda, save_dirname=None):
    if save_dirname is None:
        return

    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
    exe = fluid.Executor(place)

    # Use fluid.io.load_inference_model to obtain the inference program desc,
    # the feed_target_names (the names of variables that will be feeded 
    # data using feed operators), and the fetch_targets (variables that 
    # we want to obtain data from using fetch operators).
    [inference_program, feed_target_names,
     fetch_targets] = fluid.io.load_inference_model(save_dirname, exe)

    lod = [0, 4, 10]
    ts_word = create_random_lodtensor(lod, place, low=0, high=1)
    ts_pred = create_random_lodtensor(lod, place, low=0, high=1)
    ts_ctx_n2 = create_random_lodtensor(lod, place, low=0, high=1)
    ts_ctx_n1 = create_random_lodtensor(lod, place, low=0, high=1)
    ts_ctx_0 = create_random_lodtensor(lod, place, low=0, high=1)
    ts_ctx_p1 = create_random_lodtensor(lod, place, low=0, high=1)
    ts_ctx_p2 = create_random_lodtensor(lod, place, low=0, high=1)
    ts_mark = create_random_lodtensor(lod, place, low=0, high=1)

    # Construct feed as a dictionary of {feed_target_name: feed_target_data}
    # and results will contain a list of data corresponding to fetch_targets.
    assert feed_target_names[0] == 'word_data'
    assert feed_target_names[1] == 'verb_data'
    assert feed_target_names[2] == 'ctx_n2_data'
    assert feed_target_names[3] == 'ctx_n1_data'
    assert feed_target_names[4] == 'ctx_0_data'
    assert feed_target_names[5] == 'ctx_p1_data'
    assert feed_target_names[6] == 'ctx_p2_data'
    assert feed_target_names[7] == 'mark_data'

    results = exe.run(inference_program,
                      feed={
                          feed_target_names[0]: ts_word,
                          feed_target_names[1]: ts_pred,
                          feed_target_names[2]: ts_ctx_n2,
                          feed_target_names[3]: ts_ctx_n1,
                          feed_target_names[4]: ts_ctx_0,
                          feed_target_names[5]: ts_ctx_p1,
                          feed_target_names[6]: ts_ctx_p2,
                          feed_target_names[7]: ts_mark
                      },
                      fetch_list=fetch_targets,
                      return_numpy=False)
    print(results[0].lod())
    np_data = np.array(results[0])
    print("Inference Shape: ", np_data.shape)
    print("Inference results: ", np_data)


def main(use_cuda):
    if use_cuda and not fluid.core.is_compiled_with_cuda():
        return

    # Directory for saving the trained model
    save_dirname = "label_semantic_roles.inference.model"

    train(use_cuda, save_dirname)
    infer(use_cuda, save_dirname)


class TestLabelSemanticRoles(unittest.TestCase):
    def test_cuda(self):
        with self.scope_prog_guard():
            main(use_cuda=True)

    def test_cpu(self):
        with self.scope_prog_guard():
            main(use_cuda=False)

    @contextlib.contextmanager
    def scope_prog_guard(self):
        prog = fluid.Program()
        startup_prog = fluid.Program()
        scope = fluid.core.Scope()
        with fluid.scope_guard(scope):
            with fluid.program_guard(prog, startup_prog):
                yield


Q
Qiao Longfei 已提交
331
if __name__ == '__main__':
332
    unittest.main()