Created by: pakchoi

A part of model code:

firstword = paddle.layer.data(
    name="firstw", type=paddle.data_type.integer_value(dict_size))
secondword = paddle.layer.data(
    name="secondw", type=paddle.data_type.integer_value(dict_size))
thirdword = paddle.layer.data(
    name="thirdw", type=paddle.data_type.integer_value(dict_size))
fourthword = paddle.layer.data(
    name="fourthw", type=paddle.data_type.integer_value(dict_size))
nextword = paddle.layer.data(
    name="fifthw", type=paddle.data_type.integer_value(dict_size))

Efirst = wordemb(firstword)
Esecond = wordemb(secondword)
Ethird = wordemb(thirdword)
Efourth = wordemb(fourthword)


contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])

hidden1 = paddle.layer.fc(input=contextemb,
                          size=hiddensize,
                          act=paddle.activation.Sigmoid(),
                          layer_attr=paddle.attr.Extra(drop_rate=0.5),
                          bias_attr=paddle.attr.Param(learning_rate=1),
                          param_attr=paddle.attr.Param(
                                initial_std=1. / math.sqrt(embsize * 8),
                                learning_rate=1))


predictword = paddle.layer.fc(input=hidden1,
                              size=dict_size,
                              bias_attr=paddle.attr.Param(learning_rate=1),
                              act=paddle.activation.Softmax())

The part of inference code:

embsize = 32 
hiddensize = 256 
N = 5

def main():
    paddle.init(use_gpu=False, trainer_count=3) 
    word_dict = paddle.dataset.imikolov.build_dict()
    dict_size = len(word_dict)

    _, prediction, _= nnlm(hiddensize, embsize, dict_size)

    #with gzip.open("model_params.tar.gz", 'r') as f:
    #   parameters.from_tar(f)
    parameters = paddle.parameters.Parameters.from_tar(gzip.open("model_params.tar.gz", 'r'))

    infer_data = []
    infer_label_data = []
    cnt = 0
    for item in paddle.dataset.imikolov.test(word_dict, N)():
        infer_data.append((item[:4]))
        infer_label_data.append(item[4])
        cnt += 1
        if cnt == 100:
            break

    predictions = paddle.infer(
        output_layer = prediction,
        parameters = parameters,
        input = infer_data
    )

    for i, prob in enumerate(predictions):
        print prob, infer_label_data[i]

if __name__ == '__main__':
    main()
    

10 elements ahead of item[:4]:

[(2, 1063, 95, 353), (1063, 95, 353, 5), (95, 353, 5, 335), (353, 5, 335, 51), (5, 335, 51, 2072), (335, 51, 2072, 6), (51, 2072, 6, 319), (2072, 6, 319, 2072), (6, 319, 20, 5), (319, 2072, 5, 0)]