test_label_semantic_roles.py 12.6 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

Y
Yu Yang 已提交
15
import contextlib
16
import math
17
import numpy as np
Y
Yu Yang 已提交
18 19 20 21
import os
import time
import unittest

22 23
import paddle
import paddle.dataset.conll05 as conll05
24
import paddle.fluid as fluid
25 26 27 28

word_dict, verb_dict, label_dict = conll05.get_dict()
word_dict_len = len(word_dict)
label_dict_len = len(label_dict)
29
pred_dict_len = len(verb_dict)
30 31 32 33 34 35 36 37 38

mark_dict_len = 2
word_dim = 32
mark_dim = 5
hidden_dim = 512
depth = 8
mix_hidden_lr = 1e-3

IS_SPARSE = True
39
PASS_NUM = 10
40
BATCH_SIZE = 10
41 42 43 44 45 46 47 48 49 50

embedding_name = 'emb'


def load_parameter(file_name, h, w):
    with open(file_name, 'rb') as f:
        f.read(16)  # skip header.
        return np.fromfile(f, dtype=np.float32).reshape(h, w)


Y
Yu Yang 已提交
51 52
def db_lstm(word, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark,
            **ignored):
53
    # 8 features
54
    predicate_embedding = fluid.layers.embedding(
55
        input=predicate,
56
        size=[pred_dict_len, word_dim],
57
        dtype='float32',
58
        is_sparse=IS_SPARSE,
Y
Yu Yang 已提交
59
        param_attr='vemb')
60

61
    mark_embedding = fluid.layers.embedding(
62 63
        input=mark,
        size=[mark_dict_len, mark_dim],
64
        dtype='float32',
65 66 67 68
        is_sparse=IS_SPARSE)

    word_input = [word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2]
    emb_layers = [
69
        fluid.layers.embedding(
70 71
            size=[word_dict_len, word_dim],
            input=x,
J
jshower 已提交
72 73
            param_attr=fluid.ParamAttr(
                name=embedding_name, trainable=False)) for x in word_input
74 75 76 77 78
    ]
    emb_layers.append(predicate_embedding)
    emb_layers.append(mark_embedding)

    hidden_0_layers = [
J
jshower 已提交
79
        fluid.layers.fc(input=emb, size=hidden_dim) for emb in emb_layers
80 81
    ]

82
    hidden_0 = fluid.layers.sums(input=hidden_0_layers)
83

84
    lstm_0 = fluid.layers.dynamic_lstm(
85 86 87 88 89 90 91 92 93 94
        input=hidden_0,
        size=hidden_dim,
        candidate_activation='relu',
        gate_activation='sigmoid',
        cell_activation='sigmoid')

    # stack L-LSTM and R-LSTM with direct edges
    input_tmp = [hidden_0, lstm_0]

    for i in range(1, depth):
95
        mix_hidden = fluid.layers.sums(input=[
96 97
            fluid.layers.fc(input=input_tmp[0], size=hidden_dim),
            fluid.layers.fc(input=input_tmp[1], size=hidden_dim)
98 99
        ])

100
        lstm = fluid.layers.dynamic_lstm(
101 102 103 104 105 106 107 108 109
            input=mix_hidden,
            size=hidden_dim,
            candidate_activation='relu',
            gate_activation='sigmoid',
            cell_activation='sigmoid',
            is_reverse=((i % 2) == 1))

        input_tmp = [mix_hidden, lstm]

110
    feature_out = fluid.layers.sums(input=[
111 112
        fluid.layers.fc(input=input_tmp[0], size=label_dict_len, act='tanh'),
        fluid.layers.fc(input=input_tmp[1], size=label_dict_len, act='tanh')
113 114 115 116 117
    ])

    return feature_out


118
def train(use_cuda, save_dirname=None, is_local=True):
119
    # define network topology
Y
Yu Yang 已提交
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
    word = fluid.layers.data(
        name='word_data', shape=[1], dtype='int64', lod_level=1)
    predicate = fluid.layers.data(
        name='verb_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n2 = fluid.layers.data(
        name='ctx_n2_data', shape=[1], dtype='int64', lod_level=1)
    ctx_n1 = fluid.layers.data(
        name='ctx_n1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_0 = fluid.layers.data(
        name='ctx_0_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p1 = fluid.layers.data(
        name='ctx_p1_data', shape=[1], dtype='int64', lod_level=1)
    ctx_p2 = fluid.layers.data(
        name='ctx_p2_data', shape=[1], dtype='int64', lod_level=1)
    mark = fluid.layers.data(
        name='mark_data', shape=[1], dtype='int64', lod_level=1)
    feature_out = db_lstm(**locals())
    target = fluid.layers.data(
        name='target', shape=[1], dtype='int64', lod_level=1)
139
    crf_cost = fluid.layers.linear_chain_crf(
140 141
        input=feature_out,
        label=target,
J
jshower 已提交
142 143
        param_attr=fluid.ParamAttr(
            name='crfw', learning_rate=mix_hidden_lr))
144
    avg_cost = fluid.layers.mean(crf_cost)
145

146
    # TODO(qiao)
147
    # check other optimizers and check why out will be NAN
148
    sgd_optimizer = fluid.optimizer.SGD(
149
        learning_rate=fluid.layers.exponential_decay(
150
            learning_rate=0.01,
151 152
            decay_steps=100000,
            decay_rate=0.5,
Y
Yu Yang 已提交
153
            staircase=True))
W
Wu Yi 已提交
154
    sgd_optimizer.minimize(avg_cost)
155

156 157 158
    # TODO(qiao)
    # add dependency track and move this config before optimizer
    crf_decode = fluid.layers.crf_decoding(
159 160
        input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))

161
    train_data = paddle.batch(
J
jshower 已提交
162 163
        paddle.reader.shuffle(
            paddle.dataset.conll05.test(), buf_size=8192),
164
        batch_size=BATCH_SIZE)
165 166

    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
Y
Yu Yang 已提交
167 168 169 170 171
    feeder = fluid.DataFeeder(
        feed_list=[
            word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, predicate, mark, target
        ],
        place=place)
172
    exe = fluid.Executor(place)
173

174 175 176 177 178 179 180 181 182 183 184 185
    def train_loop(main_program):
        exe.run(fluid.default_startup_program())
        embedding_param = fluid.global_scope().find_var(
            embedding_name).get_tensor()
        embedding_param.set(
            load_parameter(conll05.get_embedding(), word_dict_len, word_dim),
            place)

        start_time = time.time()
        batch_id = 0
        for pass_id in xrange(PASS_NUM):
            for data in train_data():
186 187 188 189
                cost = exe.run(main_program,
                               feed=feeder.feed(data),
                               fetch_list=[avg_cost])
                cost = cost[0]
190 191

                if batch_id % 10 == 0:
192
                    print("avg_cost:" + str(cost))
193
                    if batch_id != 0:
J
jshower 已提交
194 195
                        print("second per batch: " + str((time.time(
                        ) - start_time) / batch_id))
196
                    # Set the threshold low to speed up the CI test
197
                    if float(cost) < 60.0:
198 199
                        if save_dirname is not None:
                            # TODO(liuyiqun): Change the target to crf_decode
J
jshower 已提交
200 201 202 203 204
                            fluid.io.save_inference_model(save_dirname, [
                                'word_data', 'verb_data', 'ctx_n2_data',
                                'ctx_n1_data', 'ctx_0_data', 'ctx_p1_data',
                                'ctx_p2_data', 'mark_data'
                            ], [feature_out], exe)
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
                        return

                batch_id = batch_id + 1

    if is_local:
        train_loop(fluid.default_main_program())
    else:
        port = os.getenv("PADDLE_INIT_PORT", "6174")
        pserver_ips = os.getenv("PADDLE_INIT_PSERVERS")  # ip,ip...
        eplist = []
        for ip in pserver_ips.split(","):
            eplist.append(':'.join([ip, port]))
        pserver_endpoints = ",".join(eplist)  # ip:port,ip:port...
        trainers = int(os.getenv("TRAINERS"))
        current_endpoint = os.getenv("POD_IP") + ":" + port
        trainer_id = int(os.getenv("PADDLE_INIT_TRAINER_ID"))
        training_role = os.getenv("TRAINING_ROLE", "TRAINER")
        t = fluid.DistributeTranspiler()
223
        t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
224 225 226 227 228 229 230 231
        if training_role == "PSERVER":
            pserver_prog = t.get_pserver_program(current_endpoint)
            pserver_startup = t.get_startup_program(current_endpoint,
                                                    pserver_prog)
            exe.run(pserver_startup)
            exe.run(pserver_prog)
        elif training_role == "TRAINER":
            train_loop(t.get_trainer_program())
232 233


234 235 236 237 238 239 240
def infer(use_cuda, save_dirname=None):
    if save_dirname is None:
        return

    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
    exe = fluid.Executor(place)

241 242 243 244 245 246 247 248 249
    inference_scope = fluid.core.Scope()
    with fluid.scope_guard(inference_scope):
        # Use fluid.io.load_inference_model to obtain the inference program desc,
        # the feed_target_names (the names of variables that will be feeded
        # data using feed operators), and the fetch_targets (variables that
        # we want to obtain data from using fetch operators).
        [inference_program, feed_target_names,
         fetch_targets] = fluid.io.load_inference_model(save_dirname, exe)

250 251 252 253 254 255 256 257 258
        # Setup inputs by creating LoDTensors to represent sequences of words.
        # Here each word is the basic element of these LoDTensors and the shape of 
        # each word (base_shape) should be [1] since it is simply an index to 
        # look up for the corresponding word vector.
        # Suppose the length_based level of detail (lod) info is set to [[3, 4, 2]],
        # which has only one lod level. Then the created LoDTensors will have only 
        # one higher level structure (sequence of words, or sentence) than the basic 
        # element (word). Hence the LoDTensor will hold data for three sentences of 
        # length 3, 4 and 2, respectively. 
K
Kexin Zhao 已提交
259
        # Note that lod info should be a list of lists.
260 261
        lod = [[3, 4, 2]]
        base_shape = [1]
K
Kexin Zhao 已提交
262
        # The range of random integers is [low, high]
K
Kexin Zhao 已提交
263
        word = fluid.create_random_int_lodtensor(
264
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
265
        pred = fluid.create_random_int_lodtensor(
266
            lod, base_shape, place, low=0, high=pred_dict_len - 1)
K
Kexin Zhao 已提交
267
        ctx_n2 = fluid.create_random_int_lodtensor(
268
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
269
        ctx_n1 = fluid.create_random_int_lodtensor(
270
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
271
        ctx_0 = fluid.create_random_int_lodtensor(
272
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
273
        ctx_p1 = fluid.create_random_int_lodtensor(
274
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
275
        ctx_p2 = fluid.create_random_int_lodtensor(
276
            lod, base_shape, place, low=0, high=word_dict_len - 1)
K
Kexin Zhao 已提交
277
        mark = fluid.create_random_int_lodtensor(
278
            lod, base_shape, place, low=0, high=mark_dict_len - 1)
279 280 281 282 283 284 285 286 287 288 289 290

        # Construct feed as a dictionary of {feed_target_name: feed_target_data}
        # and results will contain a list of data corresponding to fetch_targets.
        assert feed_target_names[0] == 'word_data'
        assert feed_target_names[1] == 'verb_data'
        assert feed_target_names[2] == 'ctx_n2_data'
        assert feed_target_names[3] == 'ctx_n1_data'
        assert feed_target_names[4] == 'ctx_0_data'
        assert feed_target_names[5] == 'ctx_p1_data'
        assert feed_target_names[6] == 'ctx_p2_data'
        assert feed_target_names[7] == 'mark_data'

J
jshower 已提交
291 292 293 294 295 296 297 298 299 300 301 302 303
        results = exe.run(inference_program,
                          feed={
                              feed_target_names[0]: word,
                              feed_target_names[1]: pred,
                              feed_target_names[2]: ctx_n2,
                              feed_target_names[3]: ctx_n1,
                              feed_target_names[4]: ctx_0,
                              feed_target_names[5]: ctx_p1,
                              feed_target_names[6]: ctx_p2,
                              feed_target_names[7]: mark
                          },
                          fetch_list=fetch_targets,
                          return_numpy=False)
304 305 306
        print(results[0].lod())
        np_data = np.array(results[0])
        print("Inference Shape: ", np_data.shape)
307 308


309
def main(use_cuda, is_local=True):
310 311 312 313 314 315
    if use_cuda and not fluid.core.is_compiled_with_cuda():
        return

    # Directory for saving the trained model
    save_dirname = "label_semantic_roles.inference.model"

316
    train(use_cuda, save_dirname, is_local)
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
    infer(use_cuda, save_dirname)


class TestLabelSemanticRoles(unittest.TestCase):
    def test_cuda(self):
        with self.scope_prog_guard():
            main(use_cuda=True)

    def test_cpu(self):
        with self.scope_prog_guard():
            main(use_cuda=False)

    @contextlib.contextmanager
    def scope_prog_guard(self):
        prog = fluid.Program()
        startup_prog = fluid.Program()
        scope = fluid.core.Scope()
        with fluid.scope_guard(scope):
            with fluid.program_guard(prog, startup_prog):
                yield


339
if __name__ == '__main__':
340
    unittest.main()
反馈
建议
客服 返回
顶部