add test file

nce_cost/README.md

-TBD
+# NCE加速词向量训练
+## 背景介绍
+在自然语言处理领域中，传统做法通常使用one-hot向量来表示词，比如词典为['我', '你', '喜欢']，可以用[1,0,0]、[0,1,0]和[0,0,1]这三个向量分别表示'我'、'你'和'喜欢'。这种表示方式比较简洁，但是当词表很大时，容易产生维度爆炸问题；而且任意两个词的向量是正交的，向量包含的信息有限。为了避免或减轻one-hot表示的缺点，目前通常使用词向量来取代one-hot表示，词向量也就是word embedding，即使用一个低维稠密的实向量取代高维稀疏的one-hot向量。训练词向量的方法有很多种，神经网络模型是其中之一，包括CBOW、Skip-gram等，这些模型本质上都是一个分类模型，当词表较大即类别较多时，传统的softmax将非常消耗时间。PaddlePaddle提供了Hsigmoid Layer、NCE Layer，来加速模型的训练过程。本文主要介绍如何使用Hsigmoid Layer来加速训练，词向量相关内容请查阅PaddlePaddle Book中的[词向量章节](https://github.com/PaddlePaddle/book/tree/develop/04.word2vec)。
+
+## NCE Layer
+NCE Layer引用自论文\[[1](#参考文献)\]，NCE是指Noise-contrastive estimation，原理是通过
+
+
+## 参考文献
+1. Gutmann, M., & Hyvärinen, A. (2010). [Noise-contrastive estimation: A new estimation principle for unnormalized statistical models](http://www.jmlr.org/proceedings/papers/v9/gutmann10a/gutmann10a.pdf). In AISTATS (Vol. 1, No. 2, p. 6).

nce_cost/infer.py

+# -*- encoding:utf-8 -*-
+import numpy as np
+import glob
+import gzip
+import paddle.v2 as paddle
+from nce_conf import network_conf
+
+
+def main():
+    paddle.init(use_gpu=False, trainer_count=1)
+    word_dict = paddle.dataset.imikolov.build_dict()
+    dict_size = len(word_dict)
+
+    prediction_layer = network_conf(
+        is_train=False,
+        hidden_size=128,
+        embedding_size=512,
+        dict_size=dict_size)
+
+    models_list = glob.glob('./models/*')
+    models_list = sorted(models_list)
+
+    with gzip.open(models_list[-1], 'r') as f:
+        parameters = paddle.parameters.Parameters.from_tar(f)
+
+    idx_word_dict = dict((v, k) for k, v in word_dict.items())
+    batch_size = 64
+    batch_ins = []
+    ins_iter = paddle.dataset.imikolov.test(word_dict, 5)
+
+    infer_data = []
+    infer_data_label = []
+    for item in paddle.dataset.imikolov.test(word_dict, 5)():
+        infer_data.append((item[:4]))
+        infer_data_label.append(item[4])
+        # Choose 100 samples from the test set to show how to infer.
+        if len(infer_data_label) == 100:
+            break
+
+    feeding = {
+        'firstw': 0,
+        'secondw': 1,
+        'thirdw': 2,
+        'fourthw': 3,
+        'fifthw': 4
+    }
+
+    predictions = paddle.infer(
+        output_layer=prediction_layer,
+        parameters=parameters,
+        input=infer_data,
+        feeding=feeding,
+        field=['value'])
+
+    for i, (prob, data,
+            label) in enumerate(zip(predictions, infer_data, infer_data_label)):
+        print '--------------------------'
+        print "No.%d Input: " % (i+1) + \
+                idx_word_dict[data[0]] + ' ' + \
+                idx_word_dict[data[1]] + ' ' + \
+                idx_word_dict[data[2]] + ' ' + \
+                idx_word_dict[data[3]]
+        print 'Ground Truth Output: ' + idx_word_dict[label]
+        print 'Predict Output: ' + idx_word_dict[prob.argsort(
+            kind='heapsort', axis=0)[-1]]
+        print
+
+
+if __name__ == '__main__':
+    main()

nce_cost/nce_conf.py

+# -*- encoding:utf-8 -*-
+import math
+import paddle.v2 as paddle
+
+
+def network_conf(hidden_size, embedding_size, dict_size, is_train):
+
+    first_word = paddle.layer.data(
+        name="firstw", type=paddle.data_type.integer_value(dict_size))
+    second_word = paddle.layer.data(
+        name="secondw", type=paddle.data_type.integer_value(dict_size))
+    third_word = paddle.layer.data(
+        name="thirdw", type=paddle.data_type.integer_value(dict_size))
+    fourth_word = paddle.layer.data(
+        name="fourthw", type=paddle.data_type.integer_value(dict_size))
+    next_word = paddle.layer.data(
+        name="fifthw", type=paddle.data_type.integer_value(dict_size))
+
+    embed_param_attr = paddle.attr.Param(
+        name="_proj", initial_std=0.001, learning_rate=1, l2_rate=0)
+    first_embedding = paddle.layer.embedding(
+        input=first_word, size=embedding_size, param_attr=embed_param_attr)
+    second_embedding = paddle.layer.embedding(
+        input=second_word, size=embedding_size, param_attr=embed_param_attr)
+    third_embedding = paddle.layer.embedding(
+        input=third_word, size=embedding_size, param_attr=embed_param_attr)
+    fourth_embedding = paddle.layer.embedding(
+        input=fourth_word, size=embedding_size, param_attr=embed_param_attr)
+
+    context_embedding = paddle.layer.concat(input=[
+        first_embedding, second_embedding, third_embedding, fourth_embedding
+    ])
+
+    hidden_layer = paddle.layer.fc(
+        input=context_embedding,
+        size=hidden_size,
+        act=paddle.activation.Tanh(),
+        bias_attr=paddle.attr.Param(learning_rate=1),
+        param_attr=paddle.attr.Param(
+            initial_std=1. / math.sqrt(embedding_size * 8), learning_rate=1))
+
+    if is_train == True:
+        cost = paddle.layer.nce(
+            input=hidden_layer,
+            label=next_word,
+            num_classes=dict_size,
+            param_attr=paddle.attr.Param(name='nce_w'),
+            bias_attr=paddle.attr.Param(name='nce_b'),
+            act=paddle.activation.Sigmoid(),
+            num_neg_samples=25,
+            neg_distribution=None)
+        return cost
+    else:
+        with paddle.layer.mixed(
+                size=dict_size,
+                act=paddle.activation.Softmax(),
+                bias_attr=paddle.attr.Param(name='nce_b')) as prediction:
+            prediction += paddle.layer.trans_full_matrix_projection(
+                input=hidden_layer, param_attr=paddle.attr.Param(name='nce_w'))
+
+        return prediction

nce_cost/train.py

+# -*- encoding:utf-8 -*-
+import paddle.v2 as paddle
+import gzip
+
+from nce_conf import network_conf
+
+
+def main():
+    paddle.init(use_gpu=False, trainer_count=1)
+    word_dict = paddle.dataset.imikolov.build_dict()
+    dict_size = len(word_dict)
+
+    cost = network_conf(
+        is_train=True, hidden_size=128, embedding_size=512, dict_size=dict_size)
+
+    parameters = paddle.parameters.create(cost)
+    adagrad = paddle.optimizer.Adam(learning_rate=1e-4)
+    trainer = paddle.trainer.SGD(cost, parameters, adagrad)
+
+    def event_handler(event):
+        if isinstance(event, paddle.event.EndIteration):
+            if event.batch_id % 1000 == 0:
+                print "Pass %d, Batch %d, Cost %f" % (
+                    event.pass_id, event.batch_id, event.cost)
+
+        if isinstance(event, paddle.event.EndPass):
+            result = trainer.test(
+                paddle.batch(paddle.dataset.imikolov.test(word_dict, 5), 64))
+            print "Test here.. Pass %d, Cost %f" % (event.pass_id, result.cost)
+
+            model_name = "./models/model_pass_%05d.tar.gz" % event.pass_id
+            print "Save model into %s ..." % model_name
+            with gzip.open(model_name, 'w') as f:
+                parameters.to_tar(f)
+
+    feeding = {
+        'firstw': 0,
+        'secondw': 1,
+        'thirdw': 2,
+        'fourthw': 3,
+        'fifthw': 4
+    }
+
+    trainer.train(
+        paddle.batch(paddle.dataset.imikolov.train(word_dict, 5), 64),
+        num_passes=1000,
+        event_handler=event_handler,
+        feeding=feeding)
+
+
+if __name__ == '__main__':
+    main()