seqToseq_net.py 7.6 KB
Newer Older
Z
zhangjinchao01 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
# edit-mode: -*- python -*-

# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import os
from paddle.trainer_config_helpers import *


def seq_to_seq_data(data_dir,
                    is_generating,
                    dict_size=30000,
                    train_list='train.list',
                    test_list='test.list',
                    gen_list='gen.list',
                    gen_result='gen_result'):
    """
    Predefined seqToseq train data provider for application
    is_generating: whether this config is used for generating
    dict_size: word count of dictionary
    train_list: a text file containing a list of training data
    test_list: a text file containing a list of testing data
    gen_list: a text file containing a list of generating data
    gen_result: a text file containing generating result
    """
    src_lang_dict = os.path.join(data_dir, 'src.dict')
    trg_lang_dict = os.path.join(data_dir, 'trg.dict')
    src_dict = dict()
    for line_count, line in enumerate(open(src_lang_dict, "r")):
        src_dict[line.strip()] = line_count
    trg_dict = dict()
    for line_count, line in enumerate(open(trg_lang_dict, "r")):
        trg_dict[line.strip()] = line_count

    if is_generating:
        train_list = None
        test_list = os.path.join(data_dir, gen_list)
        trg_dict = None
    else:
        train_list = os.path.join(data_dir, train_list)
        test_list = os.path.join(data_dir,test_list)

    define_py_data_sources2(train_list, test_list,
                           module = "dataprovider",
                           obj = "process",
                           args = {"src_dict": src_dict,
                                   "trg_dict": trg_dict})

W
wangjiang03 已提交
61
    return {"src_dict_path": src_lang_dict, "trg_dict_path": trg_lang_dict,
Z
zhangjinchao01 已提交
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
            "gen_result": gen_result}


def gru_encoder_decoder(data_conf,
                        is_generating,
                        word_vector_dim=512,
                        encoder_size=512,
                        decoder_size=512,
                        beam_size=3,
                        max_length=250):
    """
    A wrapper for an attention version of GRU Encoder-Decoder network
    is_generating: whether this config is used for generating
    encoder_size: dimension of hidden unit in GRU Encoder network
    decoder_size: dimension of hidden unit in GRU Decoder network
    word_vector_dim: dimension of word vector
    beam_size: expand width in beam search
    max_length: a stop condition of sequence generation
    """
    for k, v in data_conf.iteritems():
        globals()[k] = v
    source_dict_dim = len(open(src_dict_path, "r").readlines())
    target_dict_dim = len(open(trg_dict_path, "r").readlines())
    gen_trans_file = gen_result

    src_word_id = data_layer(name='source_language_word', size=source_dict_dim)
    src_embedding = embedding_layer(
        input=src_word_id,
        size=word_vector_dim,
W
wangjiang03 已提交
91 92
        param_attr=ParamAttr(name='_source_language_embedding'))
    src_forward = simple_gru(input=src_embedding, size=encoder_size)
Z
zhangjinchao01 已提交
93 94
    src_backward = simple_gru(input=src_embedding,
                              size=encoder_size,
W
wangjiang03 已提交
95
                              reverse=True)
Z
zhangjinchao01 已提交
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
    encoded_vector = concat_layer(input=[src_forward, src_backward])

    with mixed_layer(size=decoder_size) as encoded_proj:
        encoded_proj += full_matrix_projection(encoded_vector)

    backward_first = first_seq(input=src_backward)
    with mixed_layer(size=decoder_size,
                     act=TanhActivation(), ) as decoder_boot:
        decoder_boot += full_matrix_projection(backward_first)

    def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
        decoder_mem = memory(name='gru_decoder',
                             size=decoder_size,
                             boot_layer=decoder_boot)

        context = simple_attention(encoded_sequence=enc_vec,
                                   encoded_proj=enc_proj,
                                   decoder_state=decoder_mem, )

        with mixed_layer(size=decoder_size * 3) as decoder_inputs:
            decoder_inputs += full_matrix_projection(context)
            decoder_inputs += full_matrix_projection(current_word)

        gru_step = gru_step_layer(name='gru_decoder',
                                  input=decoder_inputs,
                                  output_mem=decoder_mem,
                                  size=decoder_size)

        with mixed_layer(size=target_dict_dim,
                         bias_attr=True,
                         act=SoftmaxActivation()) as out:
            out += full_matrix_projection(input=gru_step)
        return out

    decoder_group_name = "decoder_group"
131 132 133
    group_inputs=[StaticInput(input=encoded_vector,is_seq=True),
                  StaticInput(input=encoded_proj,is_seq=True)]

Z
zhangjinchao01 已提交
134 135 136 137 138 139
    if not is_generating:
        trg_embedding = embedding_layer(
            input=data_layer(name='target_language_word',
                             size=target_dict_dim),
            size=word_vector_dim,
            param_attr=ParamAttr(name='_target_language_embedding'))
140
        group_inputs.append(trg_embedding)
Z
zhangjinchao01 已提交
141 142 143 144 145 146 147 148

        # For decoder equipped with attention mechanism, in training,
        # target embeding (the groudtruth) is the data input,
        # while encoded source sequence is accessed to as an unbounded memory.
        # Here, the StaticInput defines a read-only memory
        # for the recurrent_group.
        decoder = recurrent_group(name=decoder_group_name,
                                  step=gru_decoder_with_attention,
149
                                  input=group_inputs)
Z
zhangjinchao01 已提交
150 151 152

        lbl = data_layer(name='target_language_next_word',
                         size=target_dict_dim)
153
        cost = classification_cost(input=decoder, label=lbl)
Z
zhangjinchao01 已提交
154 155
        outputs(cost)
    else:
156
        # In generation, the decoder predicts a next target word based on
Z
zhangjinchao01 已提交
157
        # the encoded source sequence and the last generated target word.
158

Z
zhangjinchao01 已提交
159
        # The encoded source sequence (encoder's output) must be specified by
160 161 162 163 164
        # StaticInput, which is a read-only memory.
        # Embedding of the last generated word is automatically gotten by
        # GeneratedInputs, which is initialized by a start mark, such as <s>,
        # and must be included in generation.

Z
zhangjinchao01 已提交
165 166 167 168
        trg_embedding = GeneratedInput(
            size=target_dict_dim,
            embedding_name='_target_language_embedding',
            embedding_size=word_vector_dim)
169 170
        group_inputs.append(trg_embedding)

Z
zhangjinchao01 已提交
171 172
        beam_gen = beam_search(name=decoder_group_name,
                               step=gru_decoder_with_attention,
173
                               input=group_inputs,
Z
zhangjinchao01 已提交
174 175 176 177 178 179 180 181 182
                               id_input=data_layer(name="sent_id",
                                                   size=1),
                               dict_file=trg_dict_path,
                               bos_id=0,
                               eos_id=1,
                               beam_size=beam_size,
                               max_length=max_length,
                               result_file=gen_trans_file)
        outputs(beam_gen)