test_machine_translation.py 11.2 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14
import contextlib
D
dzhwinter 已提交
15

16
import numpy as np
17
import paddle
18 19 20 21
import paddle.fluid as fluid
import paddle.fluid.framework as framework
import paddle.fluid.layers as pd
from paddle.fluid.executor import Executor
22
import unittest
23
import os
24 25 26

dict_size = 30000
source_dict_dim = target_dict_dim = dict_size
27 28
hidden_dim = 32
word_dim = 16
Q
Qiao Longfei 已提交
29 30
batch_size = 2
max_length = 8
31 32
topk_size = 50
trg_dic_size = 10000
Q
Qiao Longfei 已提交
33
beam_size = 2
34

35 36 37
decoder_size = hidden_dim


38
def encoder(is_sparse):
39
    # encoder
Q
Qiao Longfei 已提交
40
    src_word_id = pd.data(
41
        name="src_word_id", shape=[1], dtype='int64', lod_level=1)
Q
Qiao Longfei 已提交
42
    src_embedding = pd.embedding(
43 44 45
        input=src_word_id,
        size=[dict_size, word_dim],
        dtype='float32',
46
        is_sparse=is_sparse,
47 48
        param_attr=fluid.ParamAttr(name='vemb'))

Q
Qiao Longfei 已提交
49 50 51 52 53
    fc1 = pd.fc(input=src_embedding, size=hidden_dim * 4, act='tanh')
    lstm_hidden0, lstm_0 = pd.dynamic_lstm(input=fc1, size=hidden_dim * 4)
    encoder_out = pd.sequence_last_step(input=lstm_hidden0)
    return encoder_out

54

55
def decoder_train(context, is_sparse):
56
    # decoder
Q
Qiao Longfei 已提交
57
    trg_language_word = pd.data(
58
        name="target_language_word", shape=[1], dtype='int64', lod_level=1)
Q
Qiao Longfei 已提交
59
    trg_embedding = pd.embedding(
60 61 62
        input=trg_language_word,
        size=[dict_size, word_dim],
        dtype='float32',
63
        is_sparse=is_sparse,
64 65
        param_attr=fluid.ParamAttr(name='vemb'))

Q
Qiao Longfei 已提交
66
    rnn = pd.DynamicRNN()
67 68
    with rnn.block():
        current_word = rnn.step_input(trg_embedding)
Q
Qiao Longfei 已提交
69 70
        pre_state = rnn.memory(init=context)
        current_state = pd.fc(input=[current_word, pre_state],
71 72
                              size=decoder_size,
                              act='tanh')
Q
Qiao Longfei 已提交
73 74 75 76 77 78

        current_score = pd.fc(input=current_state,
                              size=target_dict_dim,
                              act='softmax')
        rnn.update_memory(pre_state, current_state)
        rnn.output(current_score)
79 80

    return rnn()
81 82


83
def decoder_decode(context, is_sparse):
Q
Qiao Longfei 已提交
84 85
    init_state = context
    array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)
86
    counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
Q
Qiao Longfei 已提交
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110

    # fill the first element with init_state
    state_array = pd.create_array('float32')
    pd.array_write(init_state, array=state_array, i=counter)

    # ids, scores as memory
    ids_array = pd.create_array('int64')
    scores_array = pd.create_array('float32')

    init_ids = pd.data(name="init_ids", shape=[1], dtype="int64", lod_level=2)
    init_scores = pd.data(
        name="init_scores", shape=[1], dtype="float32", lod_level=2)

    pd.array_write(init_ids, array=ids_array, i=counter)
    pd.array_write(init_scores, array=scores_array, i=counter)

    cond = pd.less_than(x=counter, y=array_len)

    while_op = pd.While(cond=cond)
    with while_op.block():
        pre_ids = pd.array_read(array=ids_array, i=counter)
        pre_state = pd.array_read(array=state_array, i=counter)
        pre_score = pd.array_read(array=scores_array, i=counter)

111
        # expand the recursive_sequence_lengths of pre_state to be the same with pre_score
Q
Qiao Longfei 已提交
112 113 114 115 116 117
        pre_state_expanded = pd.sequence_expand(pre_state, pre_score)

        pre_ids_emb = pd.embedding(
            input=pre_ids,
            size=[dict_size, word_dim],
            dtype='float32',
118
            is_sparse=is_sparse)
Q
Qiao Longfei 已提交
119 120

        # use rnn unit to update rnn
121
        current_state = pd.fc(input=[pre_state_expanded, pre_ids_emb],
Q
Qiao Longfei 已提交
122 123
                              size=decoder_size,
                              act='tanh')
124
        current_state_with_lod = pd.lod_reset(x=current_state, y=pre_score)
Q
Qiao Longfei 已提交
125
        # use score to do beam search
126
        current_score = pd.fc(input=current_state_with_lod,
Q
Qiao Longfei 已提交
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
                              size=target_dict_dim,
                              act='softmax')
        topk_scores, topk_indices = pd.topk(current_score, k=50)
        selected_ids, selected_scores = pd.beam_search(
            pre_ids, topk_indices, topk_scores, beam_size, end_id=10, level=0)

        pd.increment(x=counter, value=1, in_place=True)

        # update the memories
        pd.array_write(current_state, array=state_array, i=counter)
        pd.array_write(selected_ids, array=ids_array, i=counter)
        pd.array_write(selected_scores, array=scores_array, i=counter)

        pd.less_than(x=counter, y=array_len, cond=cond)

    translation_ids, translation_scores = pd.beam_search_decode(
        ids=ids_array, scores=scores_array)

    # return init_ids, init_scores

    return translation_ids, translation_scores


150
def train_main(use_cuda, is_sparse, is_local=True):
151 152 153 154 155 156
    if use_cuda and not fluid.core.is_compiled_with_cuda():
        return
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

    context = encoder(is_sparse)
    rnn_out = decoder_train(context, is_sparse)
Q
Qiao Longfei 已提交
157
    label = pd.data(
158
        name="target_language_next_word", shape=[1], dtype='int64', lod_level=1)
Q
Qiao Longfei 已提交
159
    cost = pd.cross_entropy(input=rnn_out, label=label)
160
    avg_cost = pd.mean(cost)
161

162 163 164 165
    optimizer = fluid.optimizer.Adagrad(
        learning_rate=1e-4,
        regularization=fluid.regularizer.L2DecayRegularizer(
            regularization_coeff=0.1))
W
Wu Yi 已提交
166
    optimizer.minimize(avg_cost)
167 168 169

    train_data = paddle.batch(
        paddle.reader.shuffle(
170
            paddle.dataset.wmt14.train(dict_size), buf_size=1000),
171 172
        batch_size=batch_size)

173 174 175 176
    feed_order = [
        'src_word_id', 'target_language_word', 'target_language_next_word'
    ]

177 178
    exe = Executor(place)

179 180 181
    def train_loop(main_program):
        exe.run(framework.default_startup_program())

182 183 184 185 186
        feed_list = [
            main_program.global_block().var(var_name) for var_name in feed_order
        ]
        feeder = fluid.DataFeeder(feed_list, place)

187 188 189 190
        batch_id = 0
        for pass_id in xrange(1):
            for data in train_data():
                outs = exe.run(main_program,
191
                               feed=feeder.feed(data),
192 193 194 195 196 197 198 199 200 201 202
                               fetch_list=[avg_cost])
                avg_cost_val = np.array(outs[0])
                print('pass_id=' + str(pass_id) + ' batch=' + str(batch_id) +
                      " avg_cost=" + str(avg_cost_val))
                if batch_id > 3:
                    break
                batch_id += 1

    if is_local:
        train_loop(framework.default_main_program())
    else:
203 204
        port = os.getenv("PADDLE_PSERVER_PORT", "6174")
        pserver_ips = os.getenv("PADDLE_PSERVER_IPS")  # ip,ip...
205 206 207 208
        eplist = []
        for ip in pserver_ips.split(","):
            eplist.append(':'.join([ip, port]))
        pserver_endpoints = ",".join(eplist)  # ip:port,ip:port...
209
        trainers = int(os.getenv("PADDLE_TRAINERS"))
210
        current_endpoint = os.getenv("POD_IP") + ":" + port
211 212
        trainer_id = int(os.getenv("PADDLE_TRAINER_ID"))
        training_role = os.getenv("PADDLE_TRAINING_ROLE", "TRAINER")
213
        t = fluid.DistributeTranspiler()
214
        t.transpile(trainer_id, pservers=pserver_endpoints, trainers=trainers)
215 216 217 218 219 220 221 222
        if training_role == "PSERVER":
            pserver_prog = t.get_pserver_program(current_endpoint)
            pserver_startup = t.get_startup_program(current_endpoint,
                                                    pserver_prog)
            exe.run(pserver_startup)
            exe.run(pserver_prog)
        elif training_role == "TRAINER":
            train_loop(t.get_trainer_program())
223 224


225 226 227 228 229 230 231
def decode_main(use_cuda, is_sparse):
    if use_cuda and not fluid.core.is_compiled_with_cuda():
        return
    place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()

    context = encoder(is_sparse)
    translation_ids, translation_scores = decoder_decode(context, is_sparse)
Q
Qiao Longfei 已提交
232 233 234 235 236 237 238 239 240

    exe = Executor(place)
    exe.run(framework.default_startup_program())

    init_ids_data = np.array([1 for _ in range(batch_size)], dtype='int64')
    init_scores_data = np.array(
        [1. for _ in range(batch_size)], dtype='float32')
    init_ids_data = init_ids_data.reshape((batch_size, 1))
    init_scores_data = init_scores_data.reshape((batch_size, 1))
241 242
    init_recursive_seq_lens = [1] * batch_size
    init_recursive_seq_lens = [init_recursive_seq_lens, init_recursive_seq_lens]
Q
Qiao Longfei 已提交
243

244 245 246 247
    init_ids = fluid.create_lod_tensor(init_ids_data, init_recursive_seq_lens,
                                       place)
    init_scores = fluid.create_lod_tensor(init_scores_data,
                                          init_recursive_seq_lens, place)
248

Q
Qiao Longfei 已提交
249 250 251 252 253
    train_data = paddle.batch(
        paddle.reader.shuffle(
            paddle.dataset.wmt14.train(dict_size), buf_size=1000),
        batch_size=batch_size)

254 255 256 257 258 259 260 261 262 263 264
    feed_order = ['src_word_id']
    feed_list = [
        framework.default_main_program().global_block().var(var_name)
        for var_name in feed_order
    ]
    feeder = fluid.DataFeeder(feed_list, place)

    for data in train_data():
        feed_dict = feeder.feed(map(lambda x: [x[0]], data))
        feed_dict['init_ids'] = init_ids
        feed_dict['init_scores'] = init_scores
Q
Qiao Longfei 已提交
265 266 267

        result_ids, result_scores = exe.run(
            framework.default_main_program(),
268
            feed=feed_dict,
Q
Qiao Longfei 已提交
269 270
            fetch_list=[translation_ids, translation_scores],
            return_numpy=False)
271
        print result_ids.recursive_sequence_lengths()
Q
Qiao Longfei 已提交
272 273 274
        break


275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
class TestMachineTranslation(unittest.TestCase):
    pass


@contextlib.contextmanager
def scope_prog_guard():
    prog = fluid.Program()
    startup_prog = fluid.Program()
    scope = fluid.core.Scope()
    with fluid.scope_guard(scope):
        with fluid.program_guard(prog, startup_prog):
            yield


def inject_test_train(use_cuda, is_sparse):
    f_name = 'test_{0}_{1}_train'.format('cuda' if use_cuda else 'cpu', 'sparse'
                                         if is_sparse else 'dense')

    def f(*args):
        with scope_prog_guard():
            train_main(use_cuda, is_sparse)

    setattr(TestMachineTranslation, f_name, f)


def inject_test_decode(use_cuda, is_sparse, decorator=None):
    f_name = 'test_{0}_{1}_decode'.format('cuda'
                                          if use_cuda else 'cpu', 'sparse'
                                          if is_sparse else 'dense')

    def f(*args):
        with scope_prog_guard():
            decode_main(use_cuda, is_sparse)

    if decorator is not None:
        f = decorator(f)

    setattr(TestMachineTranslation, f_name, f)


for _use_cuda_ in (False, True):
    for _is_sparse_ in (False, True):
        inject_test_train(_use_cuda_, _is_sparse_)

for _use_cuda_ in (False, True):
    for _is_sparse_ in (False, True):

        _decorator_ = None
        if _use_cuda_:
            _decorator_ = unittest.skip(
                reason='Beam Search does not support CUDA!')

        inject_test_decode(
            is_sparse=_is_sparse_, use_cuda=_use_cuda_, decorator=_decorator_)

330
if __name__ == '__main__':
331
    unittest.main()
反馈
建议
客服 返回
顶部