From 05407ac26ccb8d1a2a6f1dcc07a8556872b83e0f Mon Sep 17 00:00:00 2001
From: zhangruiqing01 <zhangruiqing01@baidu.com>
Date: Wed, 21 Dec 2016 12:45:16 +0800
Subject: [PATCH] word2vec_initial

---
 word2vec/Ngram.py                |  90 ++++++++++
 word2vec/README.md               | 273 ++++++++++++++++++++++++++++++-
 word2vec/caldis.py               |  74 +++++++++
 word2vec/data/getdata.sh         |   4 +
 word2vec/dataprovider.py         |  61 +++++++
 word2vec/image/2d_similarity.png | Bin 0 -> 24166 bytes
 word2vec/image/ngram.png         | Bin 0 -> 9327 bytes
 word2vec/paraconvert.py          | 151 +++++++++++++++++
 word2vec/train.sh                |  10 ++
 9 files changed, 662 insertions(+), 1 deletion(-)
 create mode 100644 word2vec/Ngram.py
 create mode 100644 word2vec/caldis.py
 create mode 100644 word2vec/data/getdata.sh
 create mode 100644 word2vec/dataprovider.py
 create mode 100644 word2vec/image/2d_similarity.png
 create mode 100644 word2vec/image/ngram.png
 create mode 100755 word2vec/paraconvert.py
 create mode 100644 word2vec/train.sh

diff --git a/word2vec/Ngram.py b/word2vec/Ngram.py
new file mode 100644
index 0000000..66e25ca
--- /dev/null
+++ b/word2vec/Ngram.py
@@ -0,0 +1,90 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle.trainer_config_helpers import *
+
+import math
+
+#################### Data Configure ####################
+args = {'srcText': 'data/simple-examples/data/ptb.train.txt',
+        'dictfile': 'data/vocabulary.txt'}
+
+define_py_data_sources2(
+    train_list="data/train.list",
+    test_list="data/test.list",
+    module="dataprovider",
+    obj="process",
+    args=args)
+
+batch_size = 100
+settings(
+    batch_size=batch_size,
+    regularization=L2Regularization(8e-4),
+    learning_rate=3e-3)
+
+
+dictsize = 1953
+embsize = 32
+hiddensize = 256
+
+firstword = data_layer(name = "firstw", size = dictsize)
+secondword = data_layer(name = "secondw", size = dictsize)
+thirdword = data_layer(name = "thirdw", size = dictsize)
+fourthword = data_layer(name = "fourthw", size = dictsize)
+nextword = data_layer(name = "fifthw", size = dictsize)
+
+# construct word embedding for each datalayer
+def wordemb(inlayer):
+    wordemb = table_projection(
+        input = inlayer,
+        size = embsize,
+        param_attr=ParamAttr(name = "_proj",
+            initial_std=0.001,
+            learning_rate = 1,
+            l2_rate= 0,))
+    return wordemb
+
+Efirst = wordemb(firstword)
+Esecond = wordemb(secondword)
+Ethird = wordemb(thirdword)
+Efourth = wordemb(fourthword)
+
+# concatentate Ngram embeddings into context embedding
+contextemb = concat_layer(input = [Efirst, Esecond, Ethird, Efourth])
+hidden1 = fc_layer(
+        input = contextemb,
+        size = hiddensize,
+        act = SigmoidActivation(),
+        layer_attr = ExtraAttr(drop_rate=0.5),
+        bias_attr = ParamAttr(learning_rate = 2),
+        param_attr = ParamAttr(
+            initial_std = 1./math.sqrt(embsize*8),
+            learning_rate = 1))
+
+# use context embedding to predict nextword
+predictword = fc_layer(
+        input = hidden1,
+        size = dictsize,
+        bias_attr = ParamAttr(learning_rate = 2),
+        act = SoftmaxActivation())
+
+cost = classification_cost(
+        input = predictword,
+        label = nextword)
+
+
+# network input and output
+outputs(cost)
+
+
diff --git a/word2vec/README.md b/word2vec/README.md
index 56bab09..51178fe 100644
--- a/word2vec/README.md
+++ b/word2vec/README.md
@@ -1 +1,272 @@
-TODO: Base on [this tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/tutorials/embedding_model/index_en.md).
+TODO: Base on [this tutorial](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/tutorials/embedding_model/index_en.md).
+
+# 背景介绍
+
+本章我们主要讲如何训练词语的向量表征，即词语在向量空间模型（vector space models）中的特征表示。首先我们来熟悉词向量和语言模型。
+
+- 词向量：word embedding是指将词语映射到的高维向量，如
+	<center>E(football) = [0.3, 4.2, -1.5, ...]</center>
+	<center>E(baseball) = [0.2, 5.6, -2.3, ...]</center>
+向量空间模型中，每个词/句子/文档 都有自己的向量表示，可以通过该向量来计算两个词/句子/文档之间的距离。学习词向量的目标就是希望训练出来这样一个映射矩阵，能够将相同语义的词语(如"football"和"baseball")映射到相近的特征向量。
+在信息检索中，我们可以根据query和文档关键词向量间的夹角判断相关性；在句法分析和语义分析等自然语言处理任务中，训练好的词向量可以用来初始化模型，以得到更好的效果；有了词向量之后我们同样可以用聚类的方法将文档中同义词进行分组，从而进行文档分类。
+
+- 语言模型：语言模型旨在为语句的联合概率函数建模。统计语言模型中，一句话的联合概率可以表示为其中所有词语条件概率的乘积，即
+$$P(w_1, ..., w_T) = \prod_{t=1}^TP(w_t | w_1, ... , w_{t-1})$$
+其中$w_i$表示句子中的第i个词。
+语言模型有很多应用领域，如机器翻译、语音识别、信息检索、词性标注、手写识别等。这些应用有个共同的特点，就是希望能得到一个连续序列的概率。拿信息检索为例，当你在搜索“how long is a football bame”时（bame是一个医学名词），搜索引擎会提示你是否希望搜索"how long is a football game", 这是因为根据语言模型计算出“how long is a football bame”的概率很低，而与bame近似的，可能引起typo的词中，game会使该句生成的概率最大。
+
+- 语言模型与词向量的关系：
+	在实际应用中, 语言模型和词向量密不可分。如下图所示，语言模型希望得到一句话的概率时，训练模型的输入是词语映射到的词向量。
+	<p align="center">	
+	<center>![sentence_emb](image/sentence_emb.png)</center>
+	</p>
+	相反，词向量的训练也基于语言模型。我们希望训练出来的词向量能够将相同语义的词语映射到相近的特征向量，而语言模型的训练语料中经常出现"how long is a football game" 和"how long is a baseball game"，即可通过语言模型学到相近特征的"football"和"baseball"了。本章中，我们就主要介绍语言模型和词向量的关系，以及如何训练词向量。
+	
+
+#效果展示
+
+本例中，当语言模型训练好后，我们可以用t-SNE将词语特征在二维上的投影画在下图，可见语义相关的词语（如a, the, these; big, huge）在投影上距离也很近， 语意无关的词（如say, business; decision, japan）在投影上的距离也很远。
+
+<center>![图片](image/2d_similarity.png)</center>
+
+另一方面，我们还可以通过计算词向量的cosine得到相似度, 如:
+```
+similarity: 0.899180685161
+please input two words: big huge
+similarity: 0.638160689862
+please input two words: billion million
+similarity: 0.632455899017
+please input two words: china japan
+similarity: 0.704361449565
+
+please input two words: a decision
+similarity: -0.436248234689
+please input two words: huge say
+similarity: -0.160392409963
+please input two words: from company
+similarity: -0.0997506977351
+please input two words: year he
+similarity: 0.0558745388603
+```
+以上结果可以通过运行`caldis.py`, 加载字典里的单词和对应训练特征结果得到，我们将在[应用模型](#应用模型)中详细描述用法。
+
+#模型概览
+在这里我们介绍4个训练词向量的模型。
+
+- **N-gram neural model**\[[1](#参考文献)\]
+	在计算语言学中，n-gram表示一个文本中连续的n个项。基于具体的应用场景，每一项可以是一个字母、单词或者音节。本模型中用每个n-gram的历史n-1个词语组成的内容来预测第n个词。
+
+- **Continuous Bag-of-Words model(CBOW)****\[[4](#参考文献)\]
+
+
+- **Skip-Gram model****\[[4](#参考文献)\]
+	CBOW方法中，根据上下文信息预测当前词，而skip-gram方法相反地用当前词预测上下文。CBOW的好处是对上下文词语分布进行了平滑，因此在小数据集上很有效，而skip-gram的方法得到了当前词上下文的很多样本，因此适用于大数据集。
+
+- **RNNLM**
+基于RNN的语言模型，将N-gram模型中固定输入个数输入改成序列输入，根据上文序列的RNN隐状态估计下一时刻的词语。
+
+
+
+### N-gram neural model
+n-gram模型是统计语言模型中的一种重要方法，
+
+Bengio等人在2003年的一篇经典工作A Neural Probabilistic Language Model中\[[1](#参考文献)\]提出，可以通过学习大量语料得到词语的向量表达，通过这些向量得到整个句子的概率。用这种方法学习语言模型可以克服维度诅咒（curse of dimensionality）,即训练和测试数据不同导致的模型不准。
+
+该语言模型的优化目标是最大化
+$$\frac{1}{T}\sum_t f(w_t, w_{t-1}, ..., w_{t-n+1};\theta) + R(\theta)$$
+
+其中$f(w_t, w_{t-1}, ..., w_{t-n+1})=P(w_t | w_1, ..., w_{t-1})$, 表示根据历史n-1个词得到当前词$w_t$的条件概率，$R(\theta)$表示正则项。
+
+在该论文的基础上稍加修改，本例中函数$f$的网络表示如图：
+
+<center>![图片](image/ngram.png)</center>
+
+对于每个样本，模型输入$w_{t-n+1},...w_{t-1}$, 输出第t个词时字典中|V|个词的概率，V表示训练语料词汇表（vocabulary），本例中n取5。
+
+
+根据softmax的定义，生成目标词$w_t$的概率为：
+$$P(w_t | w_1, ..., w_{t-n+1}) = \frac{e^{g_{w_t}}}{\sum_i^{|V|} e^{g_i}}$$
+其中$g_i$为predictword在第i个输出接点的值，$g_i = \theta_i^Tx + b$, $x$为hidden1层的特征, $\theta$和$b$为hidden1层到predictword层的全连接参数。
+
+整个网络的cost为多类分类交叉熵，用classification_cost层实现。
+
+用公式表示，
+
+$$J(\theta) = -\sum_{i=1}^N\sum_{c=1}^{|V|}y_k^{i}log(softmax(g_k^i))$$ 
+
+其中$y_k^i$表示第i个样本第k类的真实label(0或1)，$softmax(g_k^i)$表示第i个样本第k类softmax输出的概率。
+
+
+
+#数据准备
+	
+1. 数据概览：
+
+	在此demo中我们选用[Penn Tree Bank](http://www.fit.vutbr.cz/~imikolov/rnnlm/) (PTB)数据集。该数据集用在Mikolov的公开语言模型训练工具 Recurrent Neural Network Language Modeling Toolkit \[[3](#参考文献)\]中。数据集较小，训练速度快。该数据集统计情况如下:
+	<center>
+	| 训练数据 | 验证数据 | 测试数据 |
+	| -- | -- | -- |
+	| ptb.train.txt | ptb.valid.txt| ptb.test.txt|
+	|42068句|3370句|3761句|
+	</center>
+	
+2. 数据下载和准备: <br/>
+	执行data/getdata.sh下载该数据，并分别将训练数据和验证数据输入train.list和test.list文件中，供paddle训练时使用。<br/>
+	
+3. 提供数据给paddle:<br/>
+	在`dataprovider.py`中，我们将数据提供给paddle。
+`initializer`中进行dataprovider的初始化，其中主要包括字典的建立（在`build_dict`函数中实现）和paddle输入字段的格式定义。
+
+	```python
+	def build_dict(ftrain, fdict):
+		sentences = []
+	    with open(ftrain) as fin:
+	        for line in fin:
+	            line = ['<s>'] + line.strip().split() + ['<e>']
+	            sentences += line
+	    wordfreq = collections.Counter(sentences)
+	    wordfreq = filter(lambda x: x[1] > cutoff, wordfreq.items())
+	    dictionary = sorted(wordfreq, key = lambda x: (-x[1], x[0]))
+	    words, _ = list(zip(*dictionary))
+	    for word in words:
+	        print >> fdict, word
+	    word_idx = dict(zip(words, xrange(len(words))))
+	    logger.info("Dictionary size=%s" %len(words))
+	    return word_idx
+	
+	def initializer(settings, srcText, dictfile, **xargs):
+	    with open(dictfile, 'w') as fdict:
+	        settings.dicts = build_dict(srcText, fdict)
+	    input_types = []
+	    for i in xrange(N):
+	        input_types.append(integer_value(len(settings.dicts)))
+	    settings.input_types = input_types
+	```
+	
+	这里N为模型N-gram, `dataprovider.py`中定义N=5,大家也可以根据新的数据和需求自行调整N。但注意调整的同时要在模型配置文件中加入/减少相应输入字段。
+	接下来，在`process`函数中将数据逐一提供给paddle。
+
+	```python
+	@provider(init_hook=initializer)
+	def process(settings, filename):
+	    UNKID = settings.dicts['<unk>']
+	    with open(filename) as fin:
+	        for line in fin:
+	            line = ['<s>']*(N-1)  + line.strip().split() + ['<e>']
+	            line = [settings.dicts.get(w, UNKID) for w in line]
+	            for i in range(N, len(line) + 1):
+	                yield line[i-N: i]
+	```
+
+	具体来说，将每句话前面补上N-1个开始符号 <s> 末尾补上一个结束符号<e>，然后以N为窗口大小，从头到尾每次向右滑动窗口并生成一条数据。如"I have a dream" 一句提供了5条数据：
+	
+	> 	<s> <s> <s> <s> I 	
+	> <s> <s> <s> I have
+	> <s> <s> I have a 
+	> <s> I have a dream
+	> I have a dream <e>
+
+	在paddle训练时，每条数据的前4个词用来预测第5个词。
+
+
+
+#训练模型
+
+1. 首先将$w_t$之前的$n-1$个词 $w_{t-n+1},...w_{t-1}$通过|V|*D的矩阵映射到D维词向量（本例config中取D=32），
+	```
+	embsize = 32
+	Efirst = wordemb(firstword)  
+	Esecond = wordemb(secondword)
+	Ethird = wordemb(thirdword)  
+	Efourth = wordemb(fourthword)
+	```
+
+2. 将这n-1个词向量经过concat_layer连接成一个大向量作为文本上文特征contextemb层。
+	```
+	contextemb = concat_layer(input = [Efirst, Esecond, Ethird, Efourth])
+	```
+3. 将contextemb全连接到hidden1层作为文本隐层特征，再经过一个全连接映射到|V|维向量predictword层，并softmax得到|V|个词的生成概率。
+
+	```
+	# concatentate Ngram embeddings into context embedding
+	contextemb = concat_layer(input = [Efirst, Esecond, Ethird, Efourth])
+	hidden1 = fc_layer(
+	        input = contextemb,
+	        size = hiddensize,
+	        act = SigmoidActivation(),
+	        layer_attr = ExtraAttr(drop_rate=0.5),
+	        bias_attr = ParamAttr(learning_rate = 2),
+	        param_attr = ParamAttr(
+	            initial_std = 1./math.sqrt(embsize*8),
+	            learning_rate = 1))
+	
+	# use context embedding to predict nextword
+	predictword = fc_layer(
+	        input = hidden1,
+	        size = dictsize,
+	        bias_attr = ParamAttr(learning_rate = 2),
+	        act = SoftmaxActivation())
+	```
+
+4. 网络的loss function为多类交叉熵，在paddle中用`classification_cost`实现。
+	```
+	cost = classification_cost(
+	        input = predictword,
+	        label = nextword)
+	```
+
+5. 最后，执行sh train.sh进行模型的训练。其中指定了总共需要执行30个pass。
+	```
+	paddle train \
+	       --config Ngram.py \
+	       --use_gpu=1 \
+	       --dot_period=100 \
+	       --log_period=3000 \
+	       --test_period=0 \
+	       --save_dir=model \
+	       --num_passes=30
+	```
+
+#应用模型
+训练模型后，我们可以加载模型参数，用训练出来的词向量初始化其他模型，也可以将模型参数从二进制格式转换成文本格式进行后续应用。
+
+1. 初始化其他模型
+
+	该模型训练好的参数可以用来初始化其他模型，具体方法如下。
+	在paddle 训练命令行中，用`--init_model_path` 来定义初始化模型的位置，用`--load_missing_parameter_strategy`指定除了词向量以外，新模型其他参数的初始化策略。	注意被初始化的模型中，需要和原模型共享被初始化参数的参数名。
+	
+2. 转换二进制词向量到文本格式
+
+	我们提供了文件`paraconvert.py`用来互转paddle训练结果的二进制文件和文本格式特征文件。
+	 - 二进制转文本
+
+		```
+		python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM
+		```
+		转换后得到的文本文件中，第一行为文件信息，之后每一行都按顺序表示字典里一个词的特征，用逗号分隔。用法如：
+		```
+		python paraconvert.py --b2t -i model/pass-00029/_proj -o model/pass-00029/_proj.txt -d 32
+		```
+		
+	 - 文本转二进制
+		
+		```
+		python paraconvert.py --t2b -i INPUT -o OUTPUT
+		```
+
+3. 计算词语之间的余弦距离
+
+	两个向量之间的余弦值通常用来计算向量之间的距离。这里我们在`caldis.py`中实现不同词语的距离度量。
+	用法：
+ `python caldis.py VOCABULARY EMBEDDINGLAYER` <br/>
+其中，VOCABULARY是dataprovider中生成的字典，EMBEDDINGLAYER是模型训练出来的词向量，如
+ `python caldis.py data/vocabulary.txt model/pass-00029/_proj.txt`。
+
+
+
+#参考文献
+1. Bengio Y, Ducharme R, Vincent P, et al. [A neural probabilistic language model](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf)[J]. journal of machine learning research, 2003, 3(Feb): 1137-1155.
+2. Mikolov T, Sutskever I, Chen K, et al. [Distributed representations of words and phrases and their compositionality](http://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf)[C]//Advances in neural information processing systems. 2013: 3111-3119.
+3. Mikolov T, Kombrink S, Deoras A, et al. [Rnnlm-recurrent neural network language modeling toolkit](http://www.fit.vutbr.cz/~imikolov/rnnlm/rnnlm-demo.pdf)[C]//Proc. of the 2011 ASRU Workshop. 2011: 196-201.
+4. Mikolov T, Chen K, Corrado G, et al. [Efficient estimation of word representations in vector space\[J\]](https://arxiv.org/pdf/1301.3781.pdf). arXiv preprint arXiv:1301.3781, 2013.
+
diff --git a/word2vec/caldis.py b/word2vec/caldis.py
new file mode 100644
index 0000000..86aa534
--- /dev/null
+++ b/word2vec/caldis.py
@@ -0,0 +1,74 @@
+#!/bin/env python
+# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Example:
+    python caldis.py DICTIONARYTXT FEATURETXT
+
+Required arguments:
+    DICTIONARYTXT    the dictionary generated in dataprovider
+    FEATURETXT       the text format word feature, one line for one word
+"""
+
+import numpy as np
+from argparse import ArgumentParser
+
+def load_dict(fdict):
+    words = [line.strip() for line in fdict.readlines()]
+    dictionary = dict(zip(words, xrange(len(words))))
+    return dictionary
+
+def load_emb(femb):
+    feaBank = []
+    flag_firstline = True
+    for line in femb:
+        if flag_firstline:
+            flag_firstline = False
+            continue
+        fea = np.array([float(x) for x in line.strip().split(',')])
+        normfea = fea * 1.0 / np.linalg.norm(fea)
+        feaBank.append(normfea)
+    return feaBank
+
+def calcos(id1, id2, Fea):
+    f1 = Fea[id1]
+    f2 = Fea[id2]
+    return np.dot(f1.transpose(), f2)
+
+def get_wordidx(w, Dict):
+    if w not in Dict:
+        print 'ERROR: %s not in the dictionary' %w
+        return -1
+    return Dict[w]
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('dict',help = 'dictionary file')
+    parser.add_argument('fea', help = 'feature file')
+    args = parser.parse_args()
+
+    with open(args.dict) as fdict:
+        word_dict = load_dict(fdict)
+
+    with open(args.fea) as ffea:
+        word_fea = load_emb(ffea)
+
+    while True:
+        w1, w2 = raw_input("please input two words: ").split()
+        w1_id = get_wordidx(w1, word_dict)
+        w2_id = get_wordidx(w2, word_dict)
+        if w1_id == -1 or w2_id == -1:
+            continue
+        print 'similarity: %s' %(calcos(w1_id, w2_id, word_fea))
diff --git a/word2vec/data/getdata.sh b/word2vec/data/getdata.sh
new file mode 100644
index 0000000..c9e831f
--- /dev/null
+++ b/word2vec/data/getdata.sh
@@ -0,0 +1,4 @@
+wget http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
+tar -zxf simple-examples.tgz
+echo `pwd`/simple-examples/data/ptb.train.txt > train.list
+echo `pwd`/simple-examples/data/ptb.valid.txt > test.list
diff --git a/word2vec/dataprovider.py b/word2vec/dataprovider.py
new file mode 100644
index 0000000..ebec418
--- /dev/null
+++ b/word2vec/dataprovider.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle.trainer.PyDataProvider2 import *
+import collections
+import logging
+import pdb
+
+logging.basicConfig(
+    format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', )
+logger = logging.getLogger('paddle')
+logger.setLevel(logging.INFO)
+
+N = 5 # Ngram
+cutoff = 50
+
+def build_dict(ftrain, fdict):
+    sentences = []
+    with open(ftrain) as fin:
+        for line in fin:
+            line = ['<s>'] + line.strip().split() + ['<e>']
+            sentences += line
+    wordfreq = collections.Counter(sentences)
+    wordfreq = filter(lambda x: x[1] > cutoff, wordfreq.items())
+    dictionary = sorted(wordfreq, key = lambda x: (-x[1], x[0]))
+    words, _ = list(zip(*dictionary))
+    for word in words:
+        print >> fdict, word
+    word_idx = dict(zip(words, xrange(len(words))))
+    logger.info("Dictionary size=%s" %len(words))
+    return word_idx
+
+def initializer(settings, srcText, dictfile, **xargs):
+    with open(dictfile, 'w') as fdict:
+        settings.dicts = build_dict(srcText, fdict)
+    input_types = []
+    for i in xrange(N):
+        input_types.append(integer_value(len(settings.dicts)))
+    settings.input_types = input_types
+
+@provider(init_hook=initializer)
+def process(settings, filename):
+    UNKID = settings.dicts['<unk>']
+    with open(filename) as fin:
+        for line in fin:
+            line = ['<s>']*(N-1)  + line.strip().split() + ['<e>']
+            line = [settings.dicts.get(w, UNKID) for w in line]
+            for i in range(N, len(line) + 1):
+                yield line[i-N: i]
+
diff --git a/word2vec/image/2d_similarity.png b/word2vec/image/2d_similarity.png
new file mode 100644
index 0000000000000000000000000000000000000000..e67356dcf2156a9220780eb4c4ef98f4185a8294
GIT binary patch
literal 24166
zcmd43WmuGN_ywqdgdi#1LraT*beEK<Al)S?J+w4PcY}fw(p^IjB_*PCH<Cj)`^@kE
z|GL+;yB~J<18_0Jyzlcqao^`Y=iEfRR#SNLnBwt+2M?YoDayWi@ZjMA`0~d@2d@|(
zNX3934_)3UNIxhaq1pz&pjk<&N<DZ`6^ngi@(BEn^-)pB<-r5oZsga)KF1>S2M-Lc
zm1L#fK@In^v0$p}SAA}j2lhFRF3(Ayt52q3PR6~WP|TAJdB<qMD)UqSYdkYsJQow8
zT<8t|>&F&~7ObycxhjwP`_kG{UuVToT75io_FEj?ynSg~QFlIVrFVVXI(o6eOCO9c
z4SyWbfG>?nz{j{CjY%&%6bHoyFZ0T*a3bHt`M>h!bH)YRiPEZg1_j+hwQL`ij28p$
zyHf-?{CX|!Wv6?ylftfB#%6sns%~nS;M(a7GAm*}7azfA(xb_EyFziPZN4j~)??5>
zJ4Ws28pmw@{;(+1Yp7SC_5Jfq^M|?b)k*v(<fwFHL*d@D_4do^BA$pI(lW((2-mMz
z(W7w9Qf*$j(na5M1Bvq<QlpD1@I>^_6;j}^T(5s@?3zE4ZqH^NCkj+4cc!Y$puKjt
zSEpJqs8c4dnfyige}61f(Bpl1w5|oN<h>HXa&vJs$lc&~dv!T_*o`OAxO20fYJ^Km
zfSe=C{v>FHe$($CnkAMEi+(Nki*56?Em!|a%B)6n?)dCxY6pI%aPBDCf9Sl8xv0=<
z7!k0W`Vz0YI_uc3dhQfmO6k*kl~v=v#W!;Ib@Zh-#)7r$L1C`n<76H)b8XoB3PXE-
z+lg8|ho+vp>&5%f*bKM*x#{xee?JcUcL%xKYr8_QX9FK$cPY3dMnn#@?`|&3RMMYo
zmFo*d+ff8_YD<3iP-8gG_qjaM%9F*yqyFw;$Y6^eQMH-t-{!xtojuu~5x~OV+Q(mQ
z$wQj!G9o14FxU7Ri%&ijPAOHcksn%TJ=W!SgHUMr(%>-vZxeq_I^^+mmXOQt^+9_B
zjhL7A-b~%RZ%#^+%jQ)o;`V&8VN~LNE`?LCAil?}=QHs{o3HovQV^a`UFrfqMLH$)
z3;GiZbF<zk#HdKC?ThDvX|x4u**5bn9_Rn_f<y|JfsOra{a^if)u3zWKmwCild}b-
zgr84QFge&S(R8#QJ08*umg_Z?r-mX9!0qh(ZxCUr3AZ~&_s)Yb!&Wb@k!^(k{ki|0
z!l7uq@9Q6=oaM4-9nZWLV4lnI{kTQiUvJ)-b4Qq{$Dh8A&}{efy&m<yBN9*G1^YKs
z(DCdgKgr9ICCw78OU2Ul-iBD)TYoIRmFRx6*2vv_uqgEwgDHoK<`~?t!@%JjF%eP1
z>4zzUU~^E>MC?9T-PvzBz>Sr4mrT1{yub4p3HQ*VM=S_NQSiS?@ettiy>it9I}XZN
zjVdgnmc~Pm{P@c4#cFJr0z@PZ8Y#y7*eZNC2o?*6tHz(+++81>fwc}+fV8|O<MWRH
zzke+MpC9P|FFxt44{NZSHg(w?sUtOnb~3By*h$=;yoL7ef*Gy_r(=f3dzY6sUp~4z
zno=;-%7Y$3B8?fJ+3$pquZ$#Sx9o`|Z2&udn$*aLAcxm>vOIfzG<R{o?b@LQ9RAxy
z$F@sm(H3lEKntIfjfp&;DAThCaX`TP%ueKd&c%NHXYw+6O{PQhKzaM!)uHvdA}j=7
zdW|`OdE>SdgZ4%VHrwEMK;l_p*lM@+>ud1{6Xbt}ph$r*Z2Y*K3(tgl#L|e_O;wsq
zl<M%Y>r{W^)U9Pj-qho$J6c}6`=I^afl|P(68sbwHOuKcocCCqj-eL0NwY=t-+&Oc
zpBzzs3nI-;TBq$TqL22j5sW|}K^I13^1J<P#<mRX!DWUPNm$9d;uZfWMYYDx&rjJ*
zTfNWED9hg!zd3^wsZZ66ircDYyr?2%dZoQNk`0cP{aG)iBa{Yi-!N|pX7}q8GM)}G
zzSDnyYbf|_F6h2>L5z*MRz7YFrMHDE#*T_#4;{$S2Tzf{u$$uXoZBdQUx7<|$rptF
z<S0U<tf$9C$!=${y<KlR2m{akVVp&Z=*J*xa2)UMgOa1)hnNu(C`tR=-=2$^Aody-
z&zz=A!lpX{Q0IU4K5jqy$?m&c?|E!Vo9lN~e0y^|x?FNVS>(s6-&pB;xt_YrP`o`+
zmTBNTz_c8XFKy1b7W&n3D1VC{eQc+qohE0PjGV*~jpTgm=JNP9n!~7VZoV+eZD(To
zY;X2f*spCXFLF*;8C%PHy4qqdQC%Dnk^JZZ3VlM@dqdBj&m^1~ZysM<ZsyK@D9pBu
zB;`IcD|%O)Aa?rat;2kd|J`NrGwiFAZ8NYJYQku|`tRHGuOTfLU}X$Sbr@yaBjcXX
z1cBgu!CPw3GC>*(o4np{87*(S`ga(G9BeTDn4oLS&r)oz#tvrHjHmxIT>WTm&A%n@
z8)R^W;5L)x_8#+Ysp?0K1DOK$HpoP$TW8axd3-u)6gE2dsQl;8WcC9WWtCl_aKUnf
zk^k-A*i;Z5A3c8h(Fv^!5-k4l`)C?(_cN{1G@FOLrVhJPRqlhF4Hx%r;BfqIC9Mwp
zFkWzZ8}8i>t~>EVlRb72tW94rJd;~B<T3G$Fbt*fqM~VoLV_h2e;Uih6dVGV5&O0e
z<IMN#QN?cyui6_`J<Tw$li1$ct-vGpxu&vTvYT*TM!-d?O$i*Osg+YW5UaH-(X6Jo
z^0Ik&YjRmmz2tsI4=N(gnc0mk2CsdV0x)j;uK$%(+$=PoFL=+h2Sd#)R(}=}qD&I=
zejIV~0V^F=!3gfJ@SCyJ0v#L>K8LxN4Ydr2-WBonq_=Y!kKiq?+rI@&;h5gX!-58w
zL`1)vUAI@1Y`}19>g{KB|NhXNKaB}PSH1zK6Q^Q`ii27EU8!!}GavYFyaG*?u-m`J
zmYnnOlC0cOkptK}gd^c@dE?6GLq$ybYP^VsDHb|s?zRh4s5w`&c}AiSNk9if4r6gN
zt%P{b3^VsieysQrNfRSe43*IH@6Q5uT@6Wjhgo5FNo<LB{c!;Tu3M6$uIegTK45ai
z76Y$KK@QIGmx@8h#^E>)FE?oM!bIMQGNPg1J)9u1LUJf$if12zL^X00w#lpD++WoN
zYp_zw@D%T{Jme5@A^+I0=ssKK4k!4|sh(rm*)A7^@bKRLo~r-HDfakL)9HlX+ooZs
z3bU`{iL@`7m(?6x=;3XWE1cEY#Ozwyoq><UnoPm>h0(hqzR?r2S63UunVU|QbK{La
z)g^9SJ>Di>f3g)VS()DEwVjxC->v$NvTR5lwIn}s98c}s&!$zo<%vW|>QVV^RHF~m
z5fR@_Ie4t8WU&`PwDb;z!Syh!xIwbA!>Gy+%?*Z!AFU61&(4P9QVVNGl5mPDMJIG(
z5iti_Ri{>&<u<h6?&ZC1B$t@cP|&RDqoG`O@Ol0-ztltjCPG!Wa!4o2*mt#_B_@~r
zycJ)1myOUd-X1JqR|<#D?&%mjzBmj$VVt!j&&U_;^!mMlTXLiz3CApaA3KHH$TH+{
zGLG!^wlN2G@24fh9|`*tH6R~wD8l6Gd5k6f+4UP?*-+Cu$s*g?;=-Ja-*Jx+<vKM5
zPspFcW0zw@_75-I85dkOhOkD>B`VskY(Cz<t6PobKDb4*9L^xlA&?GVlLa%IwLQLA
z*^S$s;mb}oALOLZbA*<65tFL`yS#tJanNP&H*JyXIh1(;&HI#~q1H89q><mHP)(!1
zI+*HjlPP$QFAfRDABcqOnG4M~&l88As&;NsN7;04QAROqHM=s+HX`&KJH{IvjnuUw
z$@nazSV|H8ieX~UTNqf$yUTK>Gv8s-pOG>yKn6|5<56*0pJKXyH@-3A38Z8s!=TY-
zsm!6ps~`lAu7Ug;dde23)zARrTtb|L4)@O<Iik__7l$Uxnn*zW8H@s3;;wNzL!k8q
z{%&LtFBh3usboMxy_eh7bsd1%Q!d~lvr~;uiKn}ssmYvr-2iqz)~j4<MaH-AAvy%f
z^hgZ#^d!A1CcQHQp(8-$5X=7(Wq8_u8JXTR6l@au;BfN)H{X;8`P{Mn)^cZ}3}nCS
zA%KOq9a-Z+>YHZC^)aROzp1jHZ|3)T!T;Z=nj)x34+Ix|#$Tm!*>7gK#1M1oPXXu;
zfWvI<heQLHVScR|OVtw1k`x1{9^!iUy_v&ZuK!?>Gm#M20tm`=tpHUB_}xH3XgQv{
zifP=;@!rjg<f=a1nVjv7rri6j%v}eL`2Ql6@VNmnh-I3YMgMy-AI9TDT>v@jsYTp_
zV;O0ch>i_wt>d%3_nX*OqD5K(j1llUv6cu0M@hTNv}bqH$o~oLbOdr@omr3o;xjS(
zjGkjFv3P<ffEs4Q8T`kSYFY{MFj6l4v{$gpU$2r_P57)vPR!CoJj#hVw11Ty%7UdW
zH|FOsguT>kRK%>>TuSHEsWwmfEJz|aA-Xtf&h(+@I@7Vu36Ue_T~FKXeb8zJ^2O18
zsVx2hsKagsAfyo7T8H@=0Op_4V9}qUpM0^Ku0HEP#k=q*236ZclZ(|gNUrmz1`Q4d
zdi8cX(7I_P=0MhH_d|J20KK?(J|cxWXaW$Wuh4dLq)aNGQ2f>R`fTF6GU?8AO(DSF
z!@9NB%M+e(5GmI#bzo|%z0qQ+*xG1x<xePHRQ~&0a@`JII(AV`<vQIwd?{oAK7RVB
zd*N55N%tDjfsx;357BgZhyo1JOR4oukw7A~AJi=eh$CLJ-YIwl(R}O34%6`uCCnvd
zknuP5vW4A1K&Kb*JTmvs^7dNmk3R#jd=`YZTV8s2t`uxAnSB#Q?e3|(A>$G#GPBnQ
zQ%Ae236L0Tf4-&g#%Hf?vbpitAWPWo#}0O%C>{Yuu&DptO*tp@C;l+lJeNXkxjtbS
zcr?4C-v5dVOQDFBv+cs%=A5s#qoT)A+)sJrk;SV&O~n|9zW#S%w0_p(1*)_5a++wG
z?_n6O`v4QVM<egkRYT4&zA~Jt@F^b;8ywDOE!zc|T?{o2^UXRS|KBnGRB?rOv8ZO4
z^H~m6ZjTqvCNL?3mEShWTW+&D9sZHSy#Z?ATjc?p`%p@U%I6k8r$A-bWraykczJiN
zdgtr8Th-U4FvR~rN-hq>^nZbrPV33<n?s(!>#9Nt{>tX#m!$nG>~AhNYWMrBjT#Ce
zd1=gRCS+E1Oo7QG{I|kLLa)`Uu02-V_v*ry+2*%Js=<cRRB+JwpOrfR_ii0!Hu1fW
zx|~&D1h(RebNKfceG<C>pVlHX<GrF5d1*mEj}>tbqk!WA>@dA0d~O2(Z#-H8h&WIB
zoX4W?9iNd;TU>YOlVh_s437oR!j`*mCoFpRKxMDXqwfJs*D-p{;W}uoQkYey>!wp*
znFu0QqW_MZmw=UMd(a3Fn}I@QC{Yh3Z%nCM$7!@19@qiZZ`Fi)NS8!Ii}E#5(QytB
zxL2aQvs{v&X*!pXV?J5*yWVqO4k5id*bv(*3#~fr=WP&LiS8N8SHRnuZ)xzCY{bs5
z_1DLPKi_Y^zk636E1Oiwyn=I!oi`Bdouy<3(25^UMf+WI%Yv&vc2y#$o?kUCwL^W)
z$TLTlT>Xidmn$DBUw$-(x)(OCyehmTUCe%jO>(iS-4~zA&dI2d>-t;i3;><R7LhXO
z+h!^Rh%ev61(&(dpo1x#chnX)C*$h=gr?7mE^^G`3U0a7)1UXKi6Y?hHu_$Ss$II4
zAZ@?wO!euH!`M9<tf<7G-CZrQnzNA{ih|*CgoPvVaTx|P8h!9ILo80cs3#X@?{DKF
z+}`aJiTi0}VFC5zT_iRAg~?q0LH|!U)ruzoK7|}9yd+_GwEN{&Q40FIn1cJ;e50a#
zOi{}11cVcikC_Zzn5<52qGr^>u7`^O7QP3q=gp@vB~)XZUL#entqw)II-uz~Sv6@L
z-f@lA_v|3Z?#lFkvR}CQXIx!O6!oyPaR^uZ;z7%Lik|+TDjY6=*lhp)NNzbv6UFx!
zVR7jO>5&Np2HZ9ghCqUF^_;R!>+O}W*%r^^4Xz~(pUaYpHs|FYw_#~0sM^@dZ4=rq
z90;!40GJZ<f4sHk*+=d1Es5#vldd{~<IU0Y*YBVHyaLttyo|~b<)VT9hG;MJVma)V
zcY)ODqi`5X6DLR<HTMqBm(F|tp@VYrI!?&Na@MIY8JjutbQyk}Pwo0zgI${^_ivF#
zccM}Fs0rt5cj=obEc({^4(T;5VUPW%J|5-Ry$8$ZMUizM#)U;Q0Rezrg_kbm(oF3n
z``x&Dnov!6cJHveWIB>N#wk_j&sfCn+ZB85CyjT(_$}J4Udft8uX}LC8|$f|#s=&f
zc`y*l+5aR_{&$_u=(7X=No?5`O`cwi{4$o7*ZUBV+65eLMsmFOKTwKYDe1MGWgd>r
z1;O>geQ`+Kug~nuT(|#Cl+T}@^^oR<Sq0;pj)H5&%j`~lZMLVE?fpOH1;f7u)b|gx
z#a81Fd;$q7{zawUo8mX$-w5kr(f?KeG@>HK)PCH;zQoO;4B*-zVqk^CeQ|wf`eJDh
zt{KNkQv4Tvj#U?@BW{O5iga=$Q1+g(%JmiaULhjFU$ra?4reKVWbTs&9v`|%;@Oc@
z$eB>B-*75-E;gU4?Skhf^)(>r=I#CFoceUG&-PmTu8&704)pk{RT*6OX8s}x#Tu*8
zv%V>Ki0xl#tu=xF#xMfXsy#Bpl|8+@lZSx~;Xd=Ba9VJ2*Ulq?^?mW19uL<Um}iaS
zbpKiOZd)jE%+|3msD;mVLi#z8nCF_RdcX*}{77UiPNr19$$4RsI`tAslg+t|swP*4
zM*4IGW3@RRDY0a)U9$oTY5s>+nN2eL`?KTSY>pK`@7%1+I)TW|?_*B-j&>+hklVt~
zsLC|tpy^4liulzQ1L^a9HbVy8i@gKn{e(~gBbihabitT+qxeQD7h1iq+lI8KcfE%U
z`>nFwQtE6cM_BC28MH>)wtN=JRW4?BL>GLnwhQm>Zw6TW7s#WozW|Ps#xMZX(hD>7
z_B#?a7486G*v&V)`QhPEz5FHftmz@x)TX)JT7{`beofgvM*|K@n}A9Qxk-AfR)zBD
z@f64?r6%2>?mfis|0_Dg{r+mc_pGM4&Px0xVDa;*H^?Bn5iB_$Nt?sDky-i~s%%#4
z&Oij+3zR>1s}WMAO9hz@Ap8kmjAD~;9A0A}(B1dCKgTXLX|I6vZSkxmXL5xfYw%;v
zHLurgBS4+fyDs&B9zxQs)NX$?#2<na-+%sGp%M^1dnpsh95eCfaJhTFt4mYdeOKH2
zdaq$`Bh#@SR6u%wl@=XPg;T;aKVseIBGWb!KPA66+|G+iOJ#e@%p?dALu@Qa6>&Fj
zK`8lcl5(#>$$zWs%v_|RI8>qC`0*?E<0cGby$+b_Guzc~^Z-Y~#VrdXuOqqJKH5gE
z)nTHMzZWXkc-5Q1Z)*wmYBmx@x`S-5n=9_y#olv=wi10Fa=+P6C~pTKquRZvBi#FB
zYs?Q&RQty~gMggf*Ahs6%<*6>LN1cTQxi2(cEtB=+NurOUy8pEayU3PC=v2t)0Sns
zsErvv=Q7C_@o*ybO!5g%j)oF7z)|vX?DprHXo#%hYLoO5;+-kO1vOC_IppJJC7-XZ
z^hDk+iV-83=9vSgD-2OUdiYT<eYOga45X|&1-N<kU;q;)4iYPNYyvIV1dv(!w=kQ(
z|58}gGmS-Qqc%GG%>XM>)W;Bjbq<PqcT!;o=eF@gX4Ruj)9ld<e)i;Kt}|ME$A6au
z6PT}1oUGp78@2gdijf>x<#_g*P0vXnf!off5Zd;jNh#Q+knYQb;t^9P5Saa_Fpy7Z
zXSD)H1iXI#1ns)P7>!6iN41mH4k!b0EpiVL4geE<K0O9^E^yUhupIc7ZOs;Oivg$P
zR%!RUeIxL_X|H}()Fl4i1x;8CA-`MolfbbE9J|l6S+H>1iq;7^>c(8m9=esrVS<A#
zzXa$eK3K+t%z;Fsy7v24XS%6bX@ntd6iR^HDfetWr<*$ugeKsTTnZ04e2kIfOrrle
z;3UVzM&<!nMJAzS4xQf%-Unjp+6*+qqraNnc9P!kCXhRYBxZ%`WmWU)ug?i$@)F~$
zsmyupl=;mAK5Bck+qCdSJoF{qg7zMs_{A66KID0W%$nrK&(v{0Mr--NLxU^tE3))S
z6p$bNhmYekm+v}yR5{O;xm(R+#jlDu_3DR4t3Ku{;aM1e3QxR$z@QML!bbef2Y-;_
z{l(SkPMK^3p@8KOnMeL20P%N@Xi7FP<|oDwz`6h9^jr-@R`Ke_(tE$xwSYDP19@ti
zW&8f_z1YAxv34#%(@$dG90OF3hFwgstT7Tu{3Q78M@NNY;c};y#TWJwbQq8y2!%sR
zL5E$5RSFiF>=;|gaav~<!p^+DJ(>`Eh;^MZ8$;xS5rxQ5^Z%5jrUwZR4;oiv4q5qK
zOO-45x7*N7nA?WETbDyAMHd6KPrxbKDtFdhX6|htU}b5cvYs0MB87S{MCeFJ`xR(M
z`wkcx?XExu(c4KdQKN<?C68Er+R3JHncIVjs|Mu3>&>N+k`=g71HL373OK}TN=8r_
zV*u>g5XN7;sfXXN&dX3K5-L}?eb5SmzW?!Vfaq^!hm#}GwY+RZ<oF;AR40abxVt=k
zBd&+-@Jve%Mj4cH*DGw}zh-sgX1UOpz8LuKuO=>9F6};UGQ<r`83F1A-0~BpBWz62
zT?#gh7CK_8atHS%aJShBSXc@tA9gOPa2(4Eq4r#hFSe<y!Csktp=Cnagc+P=Z^Qt-
z*Y1uZK*-<Z=LX0sa`>2TVnQNA;Dme>O^4)+l*m%G|5040>fki1E$yIyO;n(TYKq{m
zc`l8)@as3d$=NHnfIyxBl*3*p#<kvZR|u?EBrxY}tPACk&|Dr?H3L?GZanLaY{DWw
z{LS}Qo;o_RfT4cFar-~Bb<GQkwE8EM6U~@}JF3A^c7zyJb_@u4{Hb})Kyz-iJ#hIa
zv;!1=_r3~rzlq9}5(oFPTt@1HaG%+D9_b?wruJcuAx_0lFNEJ(yuywCJIaxqLpvX$
zrm;&s7PYLcP05w)Xk_kM8C6L;tkx`k(>)?wl928EGa>-rF#U~!x)1pvrJzuXHRFQz
zmIgKdubd=R(dwBaKSa;wZk!DM?p@~=LDX>&FE1Grij-s0@6z=IWiBNPwzrYPy-bqT
zI<*8?Ac~{vY$Zj@9i;bQeSJvhpe9X4xpe4}M|~oNK{yc}#?rjE9ibLZKdoqk*pP@t
zLlJ*nO;7%Gu}}`VfW?;=t8ySxhp&!zYU4*8xdY{ES_V&S&Fwd%1-&~wJpy%n3@1E{
z=(xav!j_U(0s$v_Yg1AaC~<cob}@e%czy!qL3xO9de7tVxEG(@pgGS7b?6}CbdoVe
z&hP;|eS<?X8hSz#v08Jj|87IB+Q4*2^M%Ds?T4JwUYEQ_Pp-Na)yZ|xGE-!ktOjJ?
zL<;KNRwd$>lmda#%f{c7gYmzuT}=MmT#_#MMNb4A;U}`+=s$Bh*-#ITyEmKGyVbdN
z|ND1t3qv*9u*T8>X^H!rW3gG063Mg@&HN9ed{_qu3&wO?sz9Q2SWY(eV61~47lcb7
z`SZ7DY}M+H<G#qO?;q@2_(7boDegQJ@OL#qcm=sP++Sag)aJ>tKRVtT(LUJku(43<
zIUOqC{S%t_D}?j~JGt1E)Aj7)EA#{G9dB!%5zMGMpNVn<%GpX2t9+6wZ=haGYZQT!
zcR>?Fuy@~!QW*)I7%&BuEXV_L5Fj1b<;U2Ng_~b@{No~mEgZgUT^$cPWOOq8Ss2bn
zbTIs?L}X>s=-zfzOmGI!=T^~_Mm9vuQ8mQuZLjgJ#TbiwGN(}aJ$9c4y?ncYf7+r7
zc)JxG?6<Gp7V$A1djM#&oSt1qfkj)jL)gM3S0(=ZlP4~hY8#6`KYb)rdXpP13Ch*l
zR4fs*(m3Qz>vI@zk0UDXd9;?K_91-r0>P~ZLVci8Ta4JDq|x=oszJ2Yp=E;Ru$ZjR
z%dzH7D>!{C6+=7~m{J#)Dho<MhD0(1>8ppo)!xm);oV9we25y4Qo{5rfnBllxzz!w
z+Mp;jDqe|=G_z8j8cV<73&d*cG0|jY8q)wKHd|G#!mb3})Y`4=^wN#dcs<ch((giR
zTNTNEHqWm4VJc66EiT>D!BZW31L|r7;f<4~9`OOdrgg*6o*Pe~yIczq(EjTF_Vl3k
zb*u+K5Z*uhA@Gp!=tO=+Ey(vLA2Nrr<4f_)%x$GfzB;y^rm$GO8GKv;JNlgcu?x%Z
zK!Il(vO&=JV)}|lL_ZT^_0>_(#juu|8AX-(C5meMKD^n!x_Em~)dAlHG8*~3^h19a
zgzi}cB!)U!?O|(6Q&<9TiSBNooY+<92!UEJdr$)t-!2iNKu9dxkFdcB)xr87F<U^#
zBiSnPa(EP@C!mx&r6J{>0DL!T%JE_DuNSdBN)c?$A@X>$l18DKp<t|JVuA#shG&p7
zZhXfd_IXWF1DVl6X&uu{92&3>&!uDVZ#$YCD+?G%g^ew0_?h1Uuw7OU*qLjwU`=(Q
zDJ10qQ&iK2e|6lVI!2GI129TVG?TYl4(6E%le*eD;&Xf<c1}lv&5m#SKpKuEEdvO>
zay{7_oELE&KTu2)=4K;i2_Z*z6jGwD7l+G+E8UjZa5=df&+9$GGS$o-Amng$ZvD$i
zQ-p0(kwge$!{7#8X&C7Y`K>>ixHMn{JBlzM9ZR1t{90mc4g-al&?pxvBEVoQL0~pZ
zyaj&t5fN7!Z)l1uSVlM@OC^<rP0D45)ZE;oU0Z%;3RdV%Xsc_>sVqdSh`DaPt{7*c
z4L(3M`NpK2V(!TnJ9Q2_`Av4+Coxi!{Y2TJ9t1G9O4wkSl}CgieL`mt#&v6ea+Z|8
z^S`RgbRHaou{1t{k2AqE08O(wxa15jOmbX==hpc1k1A0nJRMMe!&15j7<q#CX)kG^
zO994tBda9*VySq-TXfubU00TQYPOzy<g3ryf#RkQ*YO2T0o`+diWL$80KM;Zhcny)
zy_W}!(Gq{eGIxq;aKKCpp^1w!MRAQDwY2=pBy(Hgg0ZM#?Mm9+D;swmw+ny*mk3J@
z^A5kQB6zURkV7l>o>2niKxeX*>F^gq)X=nY*wN1<-;}EGaF>EjCkR{OM09PW%G2{X
zTnjgI^KtE9-!BBkFNdDo-yWPv7L{NGs``VvsYSa)i?f^Eo$HL1Gb%&Vgzc?(WJ~6N
zXcaB`rlRcXamTPBz^L!UTIid@Dw_#h*RgsqKr9q`a9RTnM*&cq1seTHq0R*4UFMl)
z^i^TJTxv8#-WY*$7Xgp(yLzq2Q=b=g1|hm`*vVFd5Z|cXr##O_L>xn`&T*X~tQFtC
zMakhfxHa{h(*lIMYj*KlW7K!f$zpFQ$X0JG@B7|bqWY`pHhlKa+>8z`)nPSW?7g?t
zruRuR@dm2;q=GKssGFH+Ldr3r)75`>yRdhlIHLInl%&^xir<QD$9uEdC&z>ff|)ww
z1#IFs>W`8k9o4#|QAR<C*F3nGaPJ(w=rQr`8|B{w*^}^`?z}KN8)kkR6Sgo&E}b(=
zUjVQxWcGE0Oi}-L4X}}M;LWN$;UAuFV65V2ikuq&$~lcCZ|wGSl#A(7#NL}8qqkqB
zA2~8nN(m81=zZS&e2WEU>^e=)(VjNkt<#v}NpG+5NxfN&&~KouM6(pd8_Nptt{d50
zww%ZDtD*}`Pas9dgGZid+f27u1n>Ww82pXZ7!hXtz11KMVKUEB-|Lt=i>K*fZ-jpy
zY!UD}uPJ)3Mhl9fx3xswvewW}eUTADhu{Ki(f^QH%?DZJ+sl1^+Q4PdTtJ8)?*uGJ
zEe4g=*sp@JFn^vXGCze(1Lh!JCXRS`VE}UM1MtN}{obh|i|-((;J!wQHbsK|$IeF!
zHYufc0toA|Jh{UgJ)g-kq)Z@wxt1uQv-?MGygL2g1B;K5S;&kW&cB-{=kA#!x)&(r
z>$+q7rsO;|+7?h1>msgInIw79k=gzqETvUn@c&Nw$K-I)wV_~MF6g*0NAws-@+Ig(
z5CtTwVmiE2p<WN_k5_avvq1zZ@}Bim1Pbq})A<U+-tW@DuYkt~Cw8dj`U7vX<uC;|
zXMU{DJ4^Z1RCw9@ZAE!fL3b#jsSyLHk>ME)3mKLQv<AHRd$gdEaqOdhKLSL3YQrjT
z>--tO4jft+LKi=FKUpTIxJ^k-=heUkP5_Tj`TKXkY;cZi**L$=&INCi|L^G}Zfkoo
zUNmvw@1b{+;YSHiz%KvRyV+h{qUqJ&kYOHT?o20A&pw+5Adwn4E~cx$gFA4EK9BWY
zl`qWZT#3#Un~}*LD|yeI`P||av=09o7g_0?Ep^fGd~u{(QMCtLB+O7;3<}dK6FzHb
zH+UNEGrP37imKd_tR7q`f+)C&A`@z9i?1Z<Jj)as<E9c6zkQI`_5ip<PTh+l;#~Vy
z8CT?HdyWj9t`(lxT0ej<65ZZ1B0i`eK#`=u8ijnU-)~&exV3s#@w;yTg%d1{?N{~D
z91pn&9EE?8&)U<;|5i)DAP>!>z_M)|<Zju0p2=qwlRKMg2b^d?9@@K|Rmw0UtN<>7
z0~++kvxq*I9Dhd`Lf+_8rMGrGM9vPl#{8)LyTMg6o#$#lylnx3_pMd@91`_5h5Frk
zVFdcxv(7~^xp!f0P6J|UmE@V$E?^+KN<4H=0K@Lw;E5)!z>$RjYp|)<L^j(GnRsvH
zTX!GDQ}bYZuBit5h0f87tbpN6K|SEv5X+>AY|$#yWxG8p(JK2lsg@%;U2}%uux4rb
z@1YqVg{zp>P@J(9&gy8LG2Sl)g#>7mK9Rk?G=`BzrA$<>^H=S7tIg_rHL~oF4%ZjH
zNc)-<4R5Y8>qC!B*!0bXqFn=W`E&=Avl~jDz^l?`08Yz-Z_Kw=az0CF_=6I|{{pk@
z8%gdXV7mxY*uXM9VH~yTjiClSw}JKLk!HPsz3%MKWdHeJOgh>P4rS3~h`4zVBX#z7
zMPFKhk!3mEU?rMtb2xKSd1>o-WXTFnbx0P+G-|fnSZ~~kVmnp&^?){(QcSI-V%-u+
z3{Zv1K9fa%&qG3=Sq)0(sK#=q539Ip>>Sz=jde~&tHNr*y7pK7F5-J5cEHzBEx3?o
z2Z-?}v`T(pu1tC&Q)_@4Z914(sC0N>(uN-)2~%q9-ZKCq)x8QNjWVB6d~P+V9TX;P
zIdNf_hfSM<sp_+%y_m=?+eg1j&VHSHe^bbB^QXT7@eL;u%CYxKiE(ukB-X-OcBX>C
zvtsRN#(08zi+q9(0~OI!7Z{DV&nku0A%&|s!*r>9G{r=rdTx^4u6cp&<Sgo#12K;2
z=^{pcnt`*@?bB3=6h17QPdhNRY+;9p8YyfncR+R=yTA{ezS4SaeP!cv=GX^LsJXI_
z1bE2#`1UEg<KJXOZUR;418U!sF_3W%rodv@|Nf?gG%<2pK?Kx>&VkD0&ReJ;n)w-j
z)Z&dy<23G5<?KnBKLkQ^L$0qQf3{YtDJMU(KuU$H<^><d*-<+SVyF?8IJM6$?uc&;
zL|O?>T-0mp*{~Q0qDP--sGX?9xhjzBtPN;4Ie(`udGGCmjA02Vy72<mzvx6~JcrJO
z1BESvQ--A#%|%wD_;%B+au5U90bSCu8xMbGUo<Ji(k#e^n7^beP-}TBYY!YIccS|J
z5z)E{$-tW=jK5pmnMNEH(rIlBjGs88pXZQ*a(uo@ra(_94!QNDzDu2zgaX~dA65ki
z=Zvt|vr*7-p7mfNwT}%vs+vV+@-Viru2+3C^0fp``l}p|`+r$>6FjeKO4)qP5UreT
z(360)MClzYhyerXWB0S^_0qS6XJ_*sKXFFO{)cAengyinOuRzM<I;Xh_ruOYbW;Pd
zp4LccB5}4BOi41|n@mYZgOl%2lyI>zXJ!*~BWzqvWZ2**ysHXE<bl%tELLVn#B{#d
zaTgG+DiLmxb4c+vo?$3gVp2`G*qm-|Fh~l=BNSkTu?4$esGt~}{e5{BWC2q^>lko0
zn|<7^CkZS$8Z}W&s$DZ_yk>Jpy<N;$k$qLoLJIm+ky-N0Dqct-0<Zoq%Kb4BgzLYN
zcfli8QEgi+>1Nu)R$AwAg}&9!ds<nkb9BEcIJGu6Svg~_iuzp@4ycZ4WZ4m#eznf(
zoza*62j4}1C8Bl2Q!JqdimR%ojYD?<P%v`yD`+<*G+Pi>(qQ*i8;hD-3~E&$w9Voh
z(!=*n-jb0RHq{kJAs@~R=&esnbnBX_{e3)3G*(d>azdtv!~yMJF*aJZZV$kPY)tis
zae2MAoAGx_<`5cu2f)klZvYBcHCa|6El4Qcf7?CmJC43gnfP~q4RIN?Kv`AsfryB_
zq0u?otj)TjW0rw9I2^vH)CC!}ko`*NhcOyWqdwP_>=LZo9|DYbLG1zuf=EyR{3QJ0
z%$}$KzMZLgyhH!R$nujGy5F=3V_yK0^oCN<pDhW%fC>O7^1)%RR>_}b1;{YU7eKu(
z5T;J%l^E7)vNd4=4en%8x)L3iYRi*g=sKQEf%;t0^cy^GQhsTAX`^|kyat5HE8Ang
zd`kgLfoOkLN*D-ltq8n>=)Od1Fn`kFt>}o}R|SthIg4D4jZepH2ufBjX(&)zAdRHC
zR^3TN>Y`)bmE%U)ZU2DoQEzo+_E6AvV#xNaD!3{`#O>c&@`IZkU<H^0uBPqlVl#U)
zY6jgC^G&M-j7Zex6T}Qoy@%Wql+$m+6N)tzwf>pC{P4Wr4Tf>`XbEe{4XXO{>Hm%d
zhZmR~4+>+f&GvG<{uMhQQ05O<Pb23hhsHy&Nza~GK3M*y$yw1!^Up3Suuj}@L6kNL
z`?nn4vOiFOIe?;F`~wAoU^a=iSFidq9<(?Pc+kc1tbqP;hEKnxNFD=^n&S2~ZW8J9
zg#USTF?7Lm+lpJ3%0OCogzsS|#{3m?dDLLKs>P?a<Hkw`L_xbSit-$JxE{l|lkz^B
zK9qvq+zg+qKgZl|7uplr3!u1BA;yS>Q<h{e%Aw|-7*`y5o&NI#nzTo7j_)9wkn9x5
zpc&6Js%64(@4OSUzmYlyr@zERJl3TyVEG01TyVc%y~FJ^=N?1sP3ro<+d*8RHM-1~
z2x5t_!Ascj$T9QFPhU)<3yB?rX&{IkUbE1?vZ{rMuL@Y!+J7{TfEQ*}R6Zw71(bA7
z_Vs>D%xXrTFljdVwO_|2<6?^GjUMTOf(+PuCGZFjI6h@NC_=Y@IkH7-%Be^-Q($@8
z0J$3>*PcfM@d4IG_)){1Qxs8GDcp-X?wi>jUT?@Ij)6;Zf3jNe<qE~{U}M*)&?Z&x
zWWa^XSLu*l;0io9%h;wnv!e;x$@ZTf7kd>YlLf?UTPlJW5{xah%prXCO)-4EO|OiL
z$|A`mPgU4Ms#UfQ!cpKb`LS@uSW)LNFD&}X5b!YG=lgrt2zj}S(z)<-S%*}R-1?(s
zNq!nt6;5Iy-n;($!;5csMvTYoOM5U?)t6&|iXpxww#i$aC)PRel!bvR{$Vg7w(w8f
z*x=;qD$JGTe1+J0pagl+Ua`(nb#BgBw`g|+`k(%N|JIQ9;+C<lO6p1T1MOM!yyFbU
zafB`=4Z>ya5LO63mppHk#TM65vO!qZ_^c2U+!WSSuSjqZL^Om`YCrprf<B`msf455
z|M(W76pph1Z_NtnXRT-xUceQtRH2)pMPJ=9cC7?!A7<6lhU9=4gIKj5YA*pN+zCgo
zECdlV$0E%r(zyQdz~X5eHhj18*&QkTR{PJ-D-xZD=LYBd^Bj%~)$T*Q#6Zltn)$;N
z)`1;gyP~>bvci0+jc1OvBCdpnC}^aa#6TG_PbQ-zfE3cFhow<oYvKX%$pe<>^Qeg1
z0)um;&Tjg*SrCB+GpAc|GzEV(@XvTfV2_$S#CI$60(?xXR<b3TKBe#1?OPSI$o``Z
zPpV^G(Og+t$+-yf-;zPQCL@XbS&elC_yoDWXT_ZQ7dbcuzBuK6C()zX2h6G;{?|%6
z4OvjJcAbvUhC!={+1@ZR;SzSD$wJU3&59F~)$C*@LKV@A-u6TWXJ$@Ou0T)56ubqW
z$<`sEN4>pXCckY;-@0N!gs|k;>p(&>3d7fLj)BqTIyL3s!GIKpc2x%!i__w0uEb))
z&#dPbpCxh+q|%>T=pQQm`Z|&fhZsig4AkCk<;ULW{&XoC_dOb55kpPeH6rvs5hH{M
z(Xr<oa0*WXxL=_W+qOf_qzBKOkzlUY$d_-j4IRKMf$W(FuB-BWf>TJ+Cp=29w&n@=
zODUCxQ9)?<4fu<`*@#QMXZvZ7Z*f);Q|(0b)$Hj&=1F4jm3%PLIYSPMsC>5RvAoYH
zlG@{#S0^jYn)vVqsHQmYF?HYkE_hx=T8j_y>=;i;a&R%$?4&dgGz;=&E~#ZTQDtNR
zJU4EzTbb2xL92w54#PrHR~9f-7`DAB)8%i7BG9(c(5Zy{)G)Av#6xT1QIOW5J285n
zV0O+QdeQLAN8IF%FhA4YeFKmA^^}+Uo-=RdrEy)eSG>b|xPleA5`<0c)#IU&=4M^N
zsi?~(CZ5N4gFjQbk@B*N1@S6yW^JFrTO%<Qf_5t12N)IliSz{JzY!l`wxOQSd*n{G
zuasLM26xOn&6H}$IznhVnV-PN9wK7=&~1#E7Ni+jKqRY9i<K}mFU-;XbF}6uOok21
z+5jTWQKJCw2LZ8{yOrIdCQnEOv;>*r8B;YFl=<b}u`4T)D`lLxZ-{Q<zcpQWk=jA5
zK0O6s<nF{7Xd)3)Q>Du0`Moxf*c!`M<#zRwq1QEb#kVp7)xk4i{?mv~1XdoVM7y4n
ztu!O;e*>VGK}QRme?its+THg|<ZSBqMA2A#t1{i`ylY%uI-0Q<Lq-+@dh~^Srr_Zt
zfJzm_h_bn#DLH7bGyaH2mG)MEXNAg}hmXsr9saJi7`(as<5^=JUOi0UHR3|e>mW<@
zu_W<<Otmgr<fC#;`6~Q@Fkn#Nu-u*6++7MPLC?EYW?gX99m^v|9XxM2d<=p+6>?l;
z0**yS?O%ZxVpy`#v9=qJ+Y(Rg#Tqx|4@`P^4zTG%DCyXCiwkLLS`PMY4q!F5EmGjK
zj{%VWf7tW%|GQ6u=$^`{X9;XJmB<zbg%<Zcq!#?Y=?B@ypbPVSTbSeZL+q%Z;S3nD
zW`R-1%*Y2J2X!Y!=1Kgf2#`EwFaI+eDgpg>>g|I6y)PxNX|UMIFBt)+6;MKM=Gp@Z
zSHPqT%^R3v_CUqz5sL?WK(xpXgDIrMOa_RT<yc?v+A@PQX4L@*@i`!vWJ6aXCK{d0
zkbP2nO`AD2NF(Hb{TNe31?ssDz*E-%?2@{GMigbNaov97y;Igu3JU!RzzgpDDNg)b
z{I*a0VhNS>0B9fGzy`E(NIZlzKLX@Z4;=3yv6%6#z{l_xU_6>kV9B{qv=Ij#B<=qG
zV5Elx@s$0%2+QB*&s7}(3P<|61lu=&Nud&W3TE#?qejs@0N%^&yMWp`3r5=d4-(-@
zF+Fq>CjgaM0?9r`?{BJsdpfo#QZ)hfdy1(q9;LugJ5T$~Q6w#P{5(&E(_#z-|1%#(
zexMU4F1D*|K+faLR-$K*0<pO&fgS%l1+ueCz<%b!hK~~o@sT|$z|2k}UQGmVabEwq
zk3wwfVf^(^$@@_tb?gnqj;TrjCAZx_19%U>%Cv%3w=w@1nCPUTcbVSOYv8n*SId(&
ze?-RJ>UCNT%&6lBbq8(NBb&W_G~Pq<u{3*HXTWH(n?sN6Xxkf8pgp~e?c;Gp(o{my
z`pQt&=G3N8&lAwSbT&yV;RmC91@SX>{ZnD&zZfa#WTzE1rxFp==Wg}am^>TU5&Tpk
zw0;*~*Foj(DDU8<uQHc3Hq7bFjZHd&7kt$#`AKhT)=|9yum+tZ9NNYJmDNGyK2P<>
zGx&C10H(7*<s;Msbe$wQivoB_YHzyIIKQ6rbox<56X@XaaH1CXY2>DOU#7eM7383w
zQ=qxb5l{5+Q<y^|$)O5h&70~0rILI63RJBC9M&RThNnoBo`p-dd=3n9Cvp!V>WbDK
zpB`qgn{@$-)L;>k-JQk+%ObC4+PDQ1a5)@WgzVn~%>a*c-|09p2CfgUy;>nCN0Wbn
z_&hgwf4k@-a(4j%wuepQlWM@sDH=D45^zv8wD1S9Cp@A}xmfB9^3~{Xm4z~o9a(-I
zX!QVFOb>VvZg}adDPYmzJlF%WA2WeBC?qGAV64^Kqh%w*_GS~97bih$pe^82KN9r1
zB%RFwdrBeZ3DWy9A&4@v@&z>7-RZ~Q@UX~H^xX!s@Ujt2X!Eb=S~-_#m1P*X{nhka
z{VvDr7*C@;ClVskDCK)n1<HpRuI8e8y;&9O|BT6?#m1p*DZ_5IzHHGd)zIt1k4JO^
z2@2G1N!*Z-ey_Vq&<ZtEW*(Iyn>N{K<P@Z^#0wEb1w3KmD`a1hsk+!{QZDK%doaqa
z&Bhdb4S=vqL6J7m>}K0=Qni1M?kwYgWWXfjlmi=*<hb^|X&5;8z0Y<JZE65<^;hOQ
zva|uI(yi@_ag|E=r^&Ce7(!|U&?lHyRYlA?<Q4Cqa>+Gf(&8K1gC<aKLU16~HjRIM
zkvdz6<i0vsZGuii{mFb$F>q_aKIeovZ`vYS6m_F1Vdvk51vXLdfwSkP*EC^$xBw)G
zZx}@HazxLW!_UV9k~R1nbG4?xC@hes(TW%6Go^;DGr&r^oA~Q|(VsSF`jV+LVfpm8
zmv3-C4$?2~Fu#w9*5FR(^?sV%DvKt3q4aQ94VaCNSP%tsPU=0B%QCR#6g`J3N{7Zy
zfArEbV1e3AhBEvD=shPAlBbf4&UFpU*ndgUF<Am%X+Lw`b{>|vL*VU<<%e;nAQ(UH
zAq&pFwPd6y)mr+x+4Tcn2q^W%7ap#hcX%XS+yj!p0oZjvE-UosCzz5fHzq1^t+!?Y
zcQV&jNBd0<>xeoHj{iMK*|#s%fX;Y;Y?YuHFKjmfb{q$~8HON)9na61MD-ZV(S7ni
zA*;`=oWJ>~gqNadILy>4eKZny(tF(5*n|6W$uWS{=-<RSAcMtnhTEZZpbKmJFnt4Y
zsLa&__m}@f(4q)S$0jm3(Eb7}R)@I}-&Y6pfhFzyneh?#yP{t-k%A}@A6eQ+zX+n3
zUIX(>NNif9&>}Sn?DqQH!^4kpq;YS+NLn<&1~-ScH5P<4&m(I%u$6y6`s{syHsm*S
zuzCQIc*IT{EBB`Z`~C9?AGF9PG%}Ss=1Ud~U-2+0D_LXB(lkLB)^p#z=<bd6ANw~^
z=BQaRQO%M=<}h2oS6zRO#w@FEqvGy_QPuH#)$wzJG(sHze$$B~1CzPAc9A7Gz&0-;
zCgP=6tMwZa?~$8Jm%S;oL5#dsdNOFJA0A2OUzEY71icyU@Ww9);RI~~khQ=xr10mV
zVluD9R3iL?U&{^HNv_gMOJ#Mh%8Fy8aCCi^8*)Ji-x33TsiNL_-I?^&#+)n^PjqBQ
z;n44rGJF`aaMRYm(qm&OMdjIuZ<cTfI$2iwpm>$d46M`UW1noZ<z7VOBWWAGNk&Kd
zn_Qox2qR^5fqeuf$zxdTR!(1_BhZ@tX@Pxznf}vDfLxkBQFp?VTZqVzmTtg4Dw3-}
zRSX-&(|7T)nVp`;J+#HM<*MSvoWLU=4Ht`5K<TDK#7+`<<6<SV<Y;r@?^Vbk7A(VP
z4gVT??@7=s%AMrBrpGRT$32lyh-8%&s+!yJH=yjaRWM-~K~UUB0>^mM>$Op%g&3iY
z1H<7&%)Ceol~zyLgN0V#vCf^ut${pxuUvRG!2zpQG%L$z!ANZVe+2N}<MqMKWEzf_
zgp@4HQZGRV*PEQdTNQL)wDMzFgyq*yqz`te&V5&`h*<!1T0===RpRK9miI~LNbgAR
zuiULHQsU|QPNv!cCsbOB?1(1kuP6q+zVjeWb4{sqh_cw<KSH!GAA=^yjR5OO>|Tb>
z#^1_a0*-3sZ{Pqs%uHL6;w>!38N<k<Lba;bJRPc>X*FD8Fbb3snH>yPkF2V~$r82E
z=e!vM<X470OBWPjP5$_s{a*6=p5nZ29QN>xN%ZP9rD}$6pL#|Qz7sMMI*QA1++vz;
zz$Y2N6G%thIQn1f{SXao1dT2b(qGjU%Ia&K^RKdas|7Ar#{I|!gOBfTHYI$ejk{Kb
z6MU^vq^CY%CE|A>jfM`Y9F8jyaCRb2iK@*?YYHHM7;_rhiFvu!frgO~tAsF<ZuCci
zzB)XIxIE%@LAb&UX)f#+V&fp@j#9DsbgkgxjmVK2tU)N}N#|JL9SnF6#6<AtS>MSJ
zW9;9tIt<^2>t&Jj&e_82bm$(?pb{=Zsia~P&b7l)Nch`D0>24K<3vO%i(<g%A_Dq|
z0WsC|08YV<s+?g?SNacWIHZcdU_}{TQ{;#@XJs~E#?a4(6GzY!`dd_YyQw*P<b*v6
zicB1cAJE(N7rovsk}S0acwCQl;Q(FBsb|UWk{Jfs>3qXX&snmZ!^{{=`f088B~YbI
zCD2Kkze{QJ-zcYW?)NgpivBM92YTyf<`)~en^FiV^;iB(p;(Aw-e9q0EfGB<5kgUq
zP(#~)RlooJI&td0Big7{c*WO=r+Fp-Bc&E3|9iHWoMhv%n(!f;$_u0f;sEWEa{`(b
z-0!i2bi#-tvka^%j*Yx)Z+MOD?`#e*4siW~*G~7~d6<soh~4rlatMQJ3AQr_DxY~(
zPvDPj(R(_^LJ(0v3A@WN=ZJ(NX-di$MeTL=-ez<ptI@Awc7weIMWlI?n&VIEC4xP{
zCf+UNdH9<(NG$}DKYcA}5zI+Vn>X!rM>tP}$ctZ+SxWej<uEjcFQuTDS(ME}1l~#D
z*z7*W8Z^1#w(8J>Ng#;4h!x$;&ZNH?Qfp&Y%Q8-WmRtj%Wn<590S99)K|_Jmqq@dj
zVY$@eqHHECiEu&6ySVQ$+T#JZMNG`fX_Evxs5-y?n6lLS^+pD~H}~J-rmIG(*VhED
z!7XbE@&QX)wHf^P)Y@awyQMWhbXvXc&s2T;$@>0gYqKb$;@G%GgrM%blsaEEG5k4+
zjrsCot9D$i1~m4E#)c0`7YKOwzEz-=+ISIJz#60*UJJOaDfNr}7DrUuXn0;uXBvLg
z+dwJrDU2YeI%<!(Dw-WjZ^dSlF<9Vl?q<jMM!=f+7UZkwkGOEA(A&SPT8I_de|hie
zAJ)jaC#bZ*Fq`w6`S}+%O~{EPv-l-i{CkvG3{}T|Y&twTjVLU=O6wI*yA-K<|Js2k
zqszxuxcTebfRKZ!vM%8J2{OHfF&hj7hXVzoDwoImY}Wl^IT__^!M9C`&PE<G%*(%>
z7alw@0Fz{IS(yEqC!Fp5Q80}zQo78HL7^|(TJ^y?KrdTX5G^=oA}LH`wzXZxnJAZF
zScbKR0sbjO1%ul?50W11;P!``Ve1d1=9=CUbq(zoAJrK8*^)S@iCV?UH8yMDL|q3R
zdV!m$PK6Z4pcQLhVAHFdfixgmTw9jOorkCiw1DDYV?~nsn_FdR2;*;$2Jy0x5eFiC
zc>=kubJpEA(*J8!#;qXnf7JDg#J4t+I1cZVB%c$?*GJ6(RqVto8I7K*pfZ}%E&TR*
zzVuZJ$94sl_WbU=>RUqLGJ>3N_azh>D{S4FK>1&dj5bE>!JuruMVo_xzV=jVcQTZP
zpSZ$Xe~gVI8n9{D*Ei3?2-AHZNh=G8>)FVIF54b1P{JJc!}wX5pipkshREhm548ll
zVnnDLM6!4DMswj394<C%5uR^dVne72*6HKS@H=I*siT!;MS-YZmCNgSB-;XNCK&3>
z@n|fr-~PYYh*k!|1KCda<Gzd-I&szU4A75pJ5V-^PSMy~1)F9KV(bJ?8TnK#7!ju<
zJTtqR&<KwkbP&k?N0(CLJN~RA6!FzCOie3RKceMwEVb2zCw!F0p=76!iV^-xFlw0x
zh#7&Ja6S=J1X)Dzi`K6(In0o7L7m$%zsU}MnNS?8SXl>T2f|c^RB+YD2u=mMG)%Kv
zJRvO8r1mk0z-MJXq2Hs~L)Sts8;z`qMH6+l?<Gckx(mpFjtwmOALX(#=PRvQyg_$$
zBc_>W;#MC?s)n-0SygW|jXhjyR*mG5-9IU+?7LY;tOPfjG4Nk6D)8}9z_AX$rKsn$
z@FNp)How43zpODZSE7>99-!!>9s!;*^V(VBD!b=LYzH<#=d82&`)r*mv&NWuoG<6)
zOX^M5vQ_?m!ot#+XyaOw$bQ`au16D0*}%tS&@qwxNBu30!X;@6AKj3hzLbHtYn0~Q
zu;Ai_;kez5w1GgS(4Z6s^;)@8^EVUCx7WZtvXivfy|5k6#4W3Z{Er2@=X71)UnM&a
zrl~lhWqJnP4o|y=2s^l|baO`cj~9hfc$C`yYi1jVct*eEciWe26&O_=;-MAus%=Q>
z@nSJK1PwKhM=|5;kPCG!hRIuHJ2z!+$G07CF8`c@N#}*`HYnHcm4&9hDJAM3vYqD1
zXL8x7q%`&e5^lR9qH>%;Tmu#`<l=|nE&o<NWvWKB<O@!8NML^}@`_D?AQ@ef6a&Y2
zf!>T{=6kd>7lLqsb-i6DHHPG(;K(&zo2IIt4-p%uggu$L>F)w|wnyaUy|iTVD3zd)
zbL1ngJ+5pUpy@k}mP9Qx{b2D~Im>U&?x(afMHfyjpWK|%JDFOf9gDyXo;7rc`Fu>t
zXVtBr*@`h|^jW_h%LZ;z=GcD9o{mymq*obgB+&xP<hW__!aoib2-KmrZU-1{-!M%s
zOG%PxPaWzTid@afjv)OPYmf0AgOi=hYhMe5mSp*@W9q)sl3yWbGJjjB=5{Wq=PJK$
z5KfnumKGQK5#nWDJenmmuH03Ps(*cwzv&1ekA3p4=wV-(dV%-JosJ3>Xrsu<>JzA_
zb&p=Tl$`psnGmeNksI8yCFXIkFfaJ^D?Hj@<JtT2-Kv|SIn^}|K;ppt@TS**f#=Np
z`SU&x3{%zIqyEz;qF|1*a;JS5_i{F1T&FXbf-Y|u<33?$+qnhj%AuStrav_4X~wYm
ztHV=yNVgnA_<uBW=HF2Fe;>aRQIhSl?_08EOOmzhMcOWrHT%96BU_;)<dU)tBPOZK
zq-+@(34@WWxfo%rS;sE>a=$*#x&MIgea?OE+Yf%2bBxdVd_M2j`~7-7pU(&0jlKjr
z?WU9!Qab&mF+fa=F0itMU|N1VwT>AN?_PlTCZW@0TurzysmQOdT$nW!`Y?v#q3E}x
zTc!!#1@trtI{H3=U*EUwT%1(?{HpUFlj`~Mkte58RWF*>b6PKmH;kMvFuvSPd!%9&
zx~>r$GX7*ShljxwL_n?jENZ>vFL|qdXj#yh#g$$^t5Cb#n0X~;z18&)k|Q&@zb6=^
zFJ_R$D9FGge1a7vzTrjc;ZjS2YY<`6c_mhuy5${{Tq6H`0Orc$rgndLroy07YwC%5
zs>&|OQr7;Lp}8#2EdGPBlI~Po`2v$HjbMn=M-{gYGAtJ_RSLOLAKk#v6EJ$@B)$Th
zT7(cARMU)CX%<z5J5V%6q<F#~vb3jC;<pN5mfWoWkqJjs4?&c0JsM(-KUe98wbkh#
z;WB1y<#}^Dea;Eoic!|DbPnzF%%ank&1Dm<S-wZ^t>j3Fev6=Z$(B^)G2!Rbg9v@g
z<CVhP%yeT0>Qk?x4;NNe%m0aDCiccvu<yGhv#4a$UPRo-;fFa&CXLVQK8PD!2{vf|
z#ox09bB$`N7QQ8y6^^*zfP=(L1(jTDAY6rO8(bPoLLbf*;I$a6_z7!MbBzC1Ftomd
z_x0}@PdQ*l{GZowd;N}M=wcUO<QCEMs>EFJM)H-th6MHs+Cjclx$W;)w5zR!f94z!
zOZN&NuR#0yjMc>A&oa5qbiW&0cn9exh%Q?RUFP$p5u^>$sgM6dBh5*LK(CQjlhR;Z
zHY_}mjX%oh6S!ya-<KD>voV(XRq4x+v^n9ce&E*XyO9j-Xuh4i_?e%F=)0Npim?KL
zCY$!xabTEfw(&Bed0{Ey13uS$p)r>?(Y&p;r1O)vtveacaBhP#no+`aZZsP&F-2MI
zbG=L>^TlhCqQ(3uG*n@ku4hATghRb_Kx$=Fb7aQ#&><qB`O;*1_V+HdSn=nO(+i0;
z68BGQ8!Cnlmxn~;G1`tJV`XAQ!<G1DjLz^s^p0K%N61S}IiB&HbKkhloW7^KQeQ=T
z#N<6W*H$vxAk!~ZkVUPFbvxwHSmc0jFs5<z!<<tUHB{p?m*;!l@hDMt!Hv;uZOtXG
z^$AL>(feoFG0LrI`-Bq)H`Hg|ig|r|k<IgKpYfA|=|H44Rr{c_a-dZseIGI2I<Dtq
zpupm2z&G17*S7ey`?$b)g>#W1b>$1>+*piH<bJ5&EM>P%hj63m9gKieOXyA+dnOBu
zl~_zeo-x=nH844zr!ehYpuzhdP{S>ucDxVxl^ImeB<DR+x6jzV<4eMr4Vd4gYLCA~
zmF79$653iVk<z*UI}M-zR@*75|E;;A9NZ5*Vyc|i#p^08Bm3eE0V`j(s#OW1D)@Xe
z?P?3ZR>xYDCb#4fF*5}ZvPkTU;#eU;(Obnv=L2pMr1m5qM#Vfu>??4_<;L_Kl`%M5
zAQ=)<q$B>|b#MDE#BQeJ#yW7AS(cT-Z^(YYDP)DS;-yS*kiq4o<`hZG_qXp+Nn-mv
z)^_sT^7M{36i#kl94^S|=DaD+r6x_mqus{aF%>XC+)qK>@H94woBh}oC&g-Zsp?oN
zEK0UJU((O{XTkjJ-=KbP0k7#UvwG#K$uzr9w<p^K4%hl{C6F9)Mow=CMc%xnm~1Vs
z4_UkmHqLpdlqJ&RKgzb_U+oH+(i!m^gS8xV15Ww9jw|Oz2s&kP(J55()|4TmD0a~d
z(%Gdq?%Kw@(@e5+MES!P!7Jkb?nb0hK^_C!g;PUD1Di*Xy&M-+Qqq_xUwn+i<L9dj
zyU`)eu7)Zi&*Qw<eYXyYk?u-f;%89Ge$0mMFOa0*f3RCI(=e_6@NADoA(_N|Zcr{X
z$`l5-OHM=b_3m;@z>-0hRH8i>x-9xrbX;oi_*vHi41*w6RUu=7m0)~+N6(a2l>AE)
z^!px|hA+6CK`q)3sTlI)f%>t=<<gFs1E+Q;CaqDY^F#oTM7^Rf+hsRDow}c35xJqF
zI7KeT+647EM__m_1%0{m`jOlY%l*%M*RC{vRgniu997=_PeWeAK;eIi^ICnCE&)Ys
zHq(2`jSmIjpZvR$b<QWc!&TinzP8)lpU$WBo%+e3%{nkaeVs!7W(KGTvR!RxYmu+o
zbMI&WwtlI^xA+H^Mhz#5KUL5Sh3MYQ_Rt&CoAFocG7rd2pCoW|NEi-K2<-4u{{P@l
z{O>U@p3qLT=lIU`S5$zFtq&5JJX+EK6w!b7bRE*NcOZjR2IG(ZuKde$2tux%Iu4Y%
z@+9p&GUsV)H^jUb4zp=Q0*E<aW?9z4X9sFa)%@oc3M6m$13&{)dX62fh7m*|(BHCQ
z?g$JT!<}i<yHg#tS2Ta$^z3>ahgjc`k+~I+*$yP5WJ{RaB6Pmulq=_(7pjnqUt^Mx
z@5)4LDF`xs(ac)u0Sm3bDoQd;;Jzbcj$3Sc;8_`K=@hV+pJux(PX?C5TyCsj36lI@
z2!Z!n78bKrg->~gvg=QJp)h-7#99rC5d=5}!d6;)Kr;jjYkr#%0#r!_A)>Zn8$^Y*
zZ68QdBKiE=VCBX^<g^Dc1Y9#C^|4l<VB_E`=WokfeZnD{^&T&9=PDeNG<$q%c7taB
zEljiPEwWJ;(#Uz_0h8{o-+;CtQvSw{3Op1}J9VuNDOn)Toa}_F?S~^R72FzoDOQ@G
z6P3_73?SSwNac}|HXW=XgCXT`nY%#eA<Y`7E!&FgbN6~}Tp?PYFC<1DXo5X=Wen7$
z9`K;BSOM}wh+PnI_RE`tLqu8p5y}@(f;N-x4WWy}we!+7ph6FNxI>Q+4wi}oDNo=0
zBY1$hd;?@(fkmySD%f|Q2a$3{2?(QvwG?P4U5IN{dwr_2zQ%+#Mbhk3AOvTeJL6gi
zS934!od;43Pl+kP@EN~5Lb*D|A^n<HoEC9-&c8_@XQPy-{ZFKCq?`B~zX4*-_57#1
zTeBZP%A@jtX}RJWR*-4F`SN@Hc$ZY|NG-@+XQu{(@`Qsc-&3|%<N{m5Us<8x#4>+(
zU1FbS2`CX?_K%|y5WTGAt#wUAHR2U>w(oGMM18nj{*67<FIe(~!@!F%gnu^jvWjb=
z>^Rsf<QB{PeU&=nGLm?d8|^?=KkB2I&n3hq8WsA{Fv$nBnFjmgOpWEx@;8y;thJ*N
z-vEO~e~WUDzX0-xWmGKo;l#pBdVR$m`pYUXbiX53+1ExP8`)#~whKcYYIVV=U6_iN
zFM2hSUi4?N2@E8g1hF#6CmAYq+Cq9)(9wi(H1SdWilu@{j@BRybD_D|Pq{by<r58^
zG;N-yDtE1E)?}f5mPh#3!0aU78MnVhzq48lp`Jl>%LRhoL1tISIVC4rD;WkR5G#`e
zMOFy{*mF&wjH-qKe545CD1Rw%Gpm6fundeeP4qfREU63tGTXEppnbtT@25godKqWy
zFoa-UsDtpr#kN!Q$Im8NQ!i4Ff-qA|2x*~<s#f^Z*8n>l!8bg~>O5ax=9eQvTs>tJ
z;|a@oa?#Dz<k-kF&g7jI|GZvffB_vlV(?5Hr#~9ruDOnKxC(=@@T-B02%#d4d^Jkf
zRPLz$bZ(4FGAm1OTedn~l#McXJ1rIs(c`05<sP?8;HdpP`8DZmJM<mN*CKZOFMoVM
z82abtJMUiG{sAaHEC&oFBw>f$feLfr6a5d#q6cOEf5asHzxYc)7)I{z25$qaC5-wE
zJf@omi3&tm=jT|Ey77h#Y*H@(ft14A0pUG6U+{b$Z_h;dq4U-2wuP|Rl3-Q$0c)$i
zv<m4aVo0WNCIcMdEC>`9aNl2s!@~@MYS0fAbjJ7CV`d>AWEn$}_^a05h9lPopl*iR
z0Zm9zMIdYQQmlPVpl}y%pF`KSR3JqH5S9o!^Sy2Ld0<dpxZXg3O%}n+EjGvRhvDWx
zQaYvKxdsF=i|MX>D0UHy2zw}WNWpJdXB!N~bz;c^ki?N$L)0IDR|2Bw5kk;ID4Xdu
zKa|V#dXjIB4i-Ba24ltAK<BT131t*SenE^%D7anT!zx^Z!(dDjRd2M~S2(5^?5N0P
zq4MzU`fJ5un*r-Han!?0Um${o{4*yss%B#}9^nm;VVIubu}GGy7+99f6@Z}xLQO&}
z<JxeOKG+ZrR@Vdl>i}fbt|YK508DSc{k3Xx2m~e9wb|+QCwM^QmR<0AtS|_B04K0J
zqZ+=DY<Xer{vK_mjcGx1_>3aNkGVeA98(Dzekf@~3TiBIkrkGQQ}c7+1dT(b1uMJ-
z81(kWKO>^n9Imu&xi=C<$==;>ukl)HY-l61oybPnvaTuvvA{9J@_-14^GNic@B{fQ
zdNUWQW)QFwkW@g<Jdjd_VkBUfDX0`hA6Z`lVRxjF$|z4@k-pPN`q-Z(e|2fJK{G^7
z4k!INctKDw1A!oT^x)jpgQyHbea(Oknwno*l??~PEYyn22dr6fZ>28vf@GD$BR$nw
zyI7dQO=}e;-@}3}%ctAr@$tBIKqAe7tLqbpbbX)#kNG;lZ>giOg`1YHw5hk-SmVyL
zY43-H!uzyqElRv~N<_DN(^41taf}p-d8%x6=!M8qBejVG=vg<as3k2_$mkZsnPVUV
zsXb4#d8WW%vG{Tsw$D04@a8fkTPjh{axI;_edrh~7n>XxoI{oF+fAmU*kzg=<iU(#
zH5q5F4x18}Pvwu{M;l)WAuDytqS&iP!SWy>WA}v+S>&|E%&j1-A)`d|E2)BVzYw5O
zgQo%DfL6E@MU;pad6@b%^{wL=9hdcRYp&jeFwVG{&6xlieDKE96+kx1RKflljv3A*
zcG(qAe$P~C$vopenhX9GK7`;^uk2#;`7XdQo`w`$pKxHK>wRy<h1vClhGu(pddxqN
zf^ra0`sH?SrhCaa`DYvAgACx$uG4GGZKkt)c9TYUJPgzQD?Z^uwF|g$!u*XTFMMm*
z2pbBS%!X7yM~7_&(R}`vo-;fusX>1IM*D}xE}MUg=;8o790aJR$F4y^CAxlK8*xS;
z8nbVux2UD<buvUuowd%iz{^uN?|73xiU3Fy-{?vJVWg%YBjv|grER-$e1F&g-k<VT
z0>Y6&wXM%BxG<kQ(d1ATU#(l0RyPRFA}%L5Q^PPq;j7m+o>;=UB)n6q<OC6&#Y^Cy
zIhy<%!KPGau_C0$XBiB^v@}BiK3O!P%8B1ilm1KkCF+Lrzu+lUHCOJ66>Dk;{hqEE
z$+m!;J|Ktn(nFcu{M6k5cAi*F!)jKOX}xP3*Y%Mn3A!)o{^E$UBO&L#Y6G?mTy)2Y
zk#cZ@SO7G*`nZaBx~WqZ*n7m_`%U#w-tY#Fh^c?bjWKaL&W8W-G#KD6yNY!>VHAX%
zaBwi`t2x=yfd1DXV(kOjs2`@P`MfahBG7fQFXZaR-ao5em7?D<n8=-Al(V@qxv-|Z
z(2*P6bUl6XBU_)|;dL<?A8R$dH<6+nC|ak6K)5!Zgw%6#8qP@M-0QIP28yUw#qdk9
zD<nP1lE>f<#fs2M6r-VM(yab$<dm>VL@Us)pUu5oM>?Z~jr@jr{%bUm;YT8q_I~*K
zV48>D;n#`uZzL(*j{{f4i;Ci!!DB0>!RBrWhrg=wu84`_Kd~oIUefau8xg!=HlJ`|
z<DuR^M0xRSs?p*#^`F$}?TpO}PE76%e23#-_Yl>hD^-_xi-TWFQrr(<7aC;+@UXLN
zXWX~ZhSeQdT9r)v@M-K9YbOER)-t$Dy3%Z^0$~vr594y_KR-y$5-nlYvQrAw@6qWA
zgnUdrHoKDF>6JV{;lhYG*d7p?Ta2NDuZUGnwhFs0t9vEX)f>CoI~-~!DdJdGWhw-c
ztRi2KK~<BCcxIsrd0AN`kEL{k$QS&9bWYs#;~#YmZLNB4QTX7zUR|o1tn^XMjx8>k
zFya$dL9`9t?+)q_NfxEBI6h)N9{6h#0Mt#k&>6y_xW=H5aCmI%^t2B3h=eE-ZCKd^
zn`D|eb7Oi*PdG#AGat2Gv%EKv;kq>+jWRx3$RwyiMw*zRSB$=F>nbZG%yY+4Q^McI
zN$AJ7FAmu6JY?PJ#4*4NKHKbpJEA|KQ}&^1{`|hlw008OhB5DZtzIwm#KA99G&AvN
z3`yffoBfiAxHwH0yJ+?9pG3+E9LF~JO_FGq&!k6VTb1$#BqpAE7F9OV8B#UNm?byQ
zUz=RyPi}AMDSVN!%;V_4;2r1wtg00$a>c>OAm`$LL`LLr*=PRyh*sunW12!>!~bEd
Me?_n8qJ7kV0aV@D-2eap

literal 0
HcmV?d00001

diff --git a/word2vec/image/ngram.png b/word2vec/image/ngram.png
new file mode 100644
index 0000000000000000000000000000000000000000..d137d1d8f8c3940d93ed758cd8a4c0937c495871
GIT binary patch
literal 9327
zcmeI2XH-*Nx9@Eaih?wyLnslkAOs>1DFKN>sDcU>qy@xK#E=J(5+b4ygoLVeA|Tk1
zB3&SWKte|dNCzb}A%LMIbk5fIoOhge-1Fg%JI)>BjyukWJ+iX)oNLWBv)6C`_uL7;
zVs6NLSp4vwJ$rbOMhL4td-f86L*ft@(4&yp9|e5u#abC&+>_TK`4jkZz*XN&f6t!6
zs3TjBe*xdQ9~j-l?%8v+{`aw$=JnQj&mQmrBtrk%BfI%D`&<_r+}NTJA0d{+&*!J7
zA9)4TW*}$w2zk1Z*X3BvYpZ)SD*?COZ|&Dk?@Ml&Eq<PT_)>Dt6ZMY{R~{tyScaW$
z6i~`KbS!w7+gxIAvifs(6^8kmL`p`()!N4Mx~tP|>uKvFK7BCX`GCIB=?)k<YNc1l
z^<zoj=w?~p297hh*)d$5L?9T+Cx;q6bM>k&SOd$6nHjwP5&#cEu^akoiq;h5f^%f8
z9^c$*x8IGczz~{(pu92r+V;`)EOeHuFTEo?&XJ8JT)#YnBX2EbUPkEA`H{AdRyK6$
zdK&k2Yu+9gFf(wjbM+d&;2_%68`>lVxc^mh{qpa&C(ur~S<-T-VvxUrm?OKS8ioH^
zF<=PiFl`q%nnV)rvNC9t(zEy<jS0dZHR{__G+d`U$>jO@`F5YlmKT?ZO@H4Lm&c}t
z2eHN**VW%xknQ5z?z(oSrzLc~i8R=GF%-ALw^h!b?nt{GMDbsp-70E||Lrjx-l3kY
z?;xI_Y;PABH=2r|I}AS<$+xeeC=T#x2qzOvgL5EqVP-4BDnyfnb4rwvP~H4Hm8EN?
zl@6&5Z!}JJLCaZ9XF7N=4yFa~kBdo|p%CB9R;-OfY6<3P7fJxTXqnDbm|Eq7A73(G
zgR0F!P!q`*B}+RPmC^ckI<-aON&mTHXKZSrt^U+QmoJi64odF#i8N%3t-cL(Zt*RR
zU=J5hXd2pEQd!*{hB=i$Jc5v51HVjK<?}(*8|`3suDa5Sd{vlU?gQ_e+@5lfc-lol
z2aqLgx;)}eLZs8B-fjg&PUE_Z(G3rg)fK7fgkEf87acl&gHZ(|OS(Hf%}l?67*>s`
za{p?-5Jy>)-c-m5oDyG3n-jowd68g@D0NE^uAzLw-;(C@Ds{0zmm1~)Po@%j6fVT^
zLM*YF5zR>_?FYElq~lUcVX$9nP4mVr4SJR3?%*HMzkVj{2u$m5J2n$Kl|1+97UgRm
z4eC~KAotMJQ?G)R*#@_R*g0>|(br(+d49-L<w9<P4QX#f#Z<pbxo*kMsa95mdo=A*
zh0;70CyCd+OrR|BK4WqvQ;j6myO6nx4w)}~6H6Lu-ghpMVkp*TQ=dF@9=>VCyn1DN
zx#YlTcPQgf{VS<WVxpule8Y7UGPC{YaF)DqDhR15lhdG8zdn6q|FrR0H-uFuhIi%+
z2SgJ<M50pp=RPUtS-_<Cwb&5l2kfB4m>U{h!OarB?X?FYE4LrdW4j}fcS0ukxu4*&
z#7i2YGtLS6T}d`g>UZPbDp1Er$_`F#f2#4xsjUdDTbas;(y4|L(|22yXsJRfC>h#|
z3A3AG_|h{(@T>XvnwIICPJxp_Z(C{gg%6bWk-~J{sQi6u)KunCV4=AwGn27!X%QEp
zcVY+PeD5sD?EO0Z+4+`2sHvn)``aZc@1)3nd-S8QlAFGgwSIXQY-B3(RH=;EUCbC}
z=Lv|G5O*5VgiOBBaK66bU4(n$srC~Fek;f-`KiL^O?||N6(Dvyq$_k@bF@EnzTR2U
z5OQv<?IGkop9J>P9blgn-p`~>IDdi&Bozcss|meWQfw7cjU*uNu%(*QN<UZ&@C)h0
za*ttEJ85R*z)tRrmyu2>MrV~gTNtk*u@R{0L>1`{_;_9E-l6V;^L<IJmNRJa)cv&Q
z<AZ+%D#TnB!A#kz6=$Tc^~i8}n50zcZ02acs$z!C>EE=)4?c*jP@bI$+Zq2n#ZCpS
zu2(r%c0n(HF9`3Lh-en(PM*-e=}~Vndn$@s?6RQmn-e{g4WADy=y?9DF}()sVyTgo
z72^aRXctG7DtXF~_G8k)wK69Q1~C(#tM2gmCtc<%pK>{SNj*FVvM(iZ20U9O7oCN*
zGL>JLV073`fM22izAGdo71yn@fe|zfd7*ITo%adFPiQESWUA8zQLu(iW=X)LPyS@Z
zHz5t^{HMM`5!P8~!m)eeav!>&h{FKgdV%&|$)Qbdzo7|$@H_xofTy3t0muTRya_<h
z>J^%hd-;Fgy-rIy$O$d3yz@Q#AM~~24-G~*!%gVroDq<<adobTc!_`ZL$f4YPRz*&
zE8|nyXvOS!x0g8?@mB9|CWl%Xzq^YT2fCC0n@+fS{|55vR2lr*N3UMKv1Os<pwYFf
z9Gf{ylN846M?T$8+Hc6)FD$MNDV8X8LwhFthc@AA76qK`rL126zZbROnqCG&w{Qe?
z_<C)=_1r}zPtpg_W(B9sZgsHl`ds{=0$SY7yD`o0<2i(2P~#c<J0Dv!$@L2hw+AI{
zBp0K$);h2KhgFP&pU1Ti5^~n3!^BKXQl!TNNt@+n?<)0^Lz8WXV^EUXs&X0f)*#Ey
zmOU)OX|J%o1xh~JhsSj;`T34@hZ@?&7ugR!3ocX|y3)}C2HE?a&7KLH$KKTb^c+|9
zlXCFoHexK_Z(FC8UAV8+MUhi)K_X%aobxNqlmXHr?4|K4kZA>x=GfO#zEj|ff~X$D
zrM5*iqxeFpNrL+N0wtT*rC2t2ImuZfTo@bf`8I0ah`PRJ+{l_zEp>&*)DEPpV{9c+
z9wMInm!mn2zQ-;L6lUT{X%|RZ9bvd7wNjh+Wl-Mp@8pov2=mNV-=*Pa3BE(%%Yx45
zvdDqI*WfI5hOcGaT#n=)i}efa$q$Gz7wp_5V%zTO;U{4k-707yBpGMb=}U^xok{5Q
z+tdVQ=R^Zh9}nk?BP_-^ERR6KQ?;hoNcZA|d_8>Q)nx_$9?me-8JezKRf$P`YvL#h
zV=coZ1fKWmor_p9+F)vd;M>%~ZV={2YNjqpDdE)l(%&(9$08DqDlW+APrWdlhaAOC
z%##LluGxjl#6I$T7JZ%Fth*dkU*)z*Jbp-_k%E;Pa#lNioXh6JRkyRiFK!hxVHD3w
zn-K4uJ{48zDtHzNm2SUsS!WC#HQu4+RhGv)*%gW_;$2-|kgk93e~c=GL-;95T$EIX
z{|xJlFOu@?2hk9I9u_)29tOAt2{rEhC<&gY^I7ShpXyGXe+ERDBv-ej)?)M|{*o#f
z7W<%;BiVIMP}9e+@|!xw*BrLl?7QTsy}l;*BQ6#DiLaNGFrRgHgSJPp6&E%Gm#|UX
z$VjJriO)>`oE&rNd=8X1TV1hjpW55rQ$B*g?kbTjH2$j6GooN-@GiWO@o8(u_+2IP
zzheu2yG6fZjC;g>Y488~R$fou=^`kvKt=n1)@^{00|(zM0Mj%P6EXmI_7xFaEB;-y
z9RQg0<c7=BQBJe6vKs;npiB5)RU6P_H&vu@UrL2%!p*=L4mh;-Sc}QpNXYqqg*s3F
zhinVUk=5#C*!$mScXjG$^;POFHD=!ooGHv#YPEmF%D&_tGXlU3IbmP13R(-7L;!7o
ziE{uS1%V;`0WLlTcqk3TngMXo@P9qsU;qm6M~m+$m&qOc?X?c#=mg9?1K0;F!ssFZ
z(I`NHeuW<OW-|u{Mn^}PTkFdaf&(s4A}<CIi330BZ>Hl|ZAoeo-Rk{ZcPN2=mR8Df
zKO#NFMb6oZ23FIaVw%b;{FbLOUz1QK`^-@LZL9r#bh!b&r@q^kHq7s|IoFr3Y&`3t
znp$WU;*MJwu4pW4#{!ab;N8_X!eiEq{#ZSF{~j=jads6#@XqXOd3fCVK?6f^?a{2v
z@UUhnkY9yR?nlGfy1rqPBW@o(L}c?CI0ZDN51M)Cg55_<!lPr2g$>3JdJ8ezPFi{L
z8ppLS?2|@Yc36}WeR(j|S?Tkwy?P4?WWQm{P$&3aR6GSGL24{K0mwpG)9;CIWfc^o
zd7Lt*!ljxOPKw7;vrQ6&$L`0qdOJ09XINWb1jRVFpd7VgWhRIHVsl>&gQkz+9(2^b
zpP+Tdst!y#H%$|+_ZX}sod3HKV@r1+3ltFyYGJ*juKaxCVl;7#+~t*L{V$j0gRwge
zN$SAXa+zFzx$W(lW&cVWe{jQ5f^7iHG=q&g`*s0KJ#gjrtXdsX6&#v+&pkZH^k{Tz
zzrscfcalFlEe?@ms_G%ZC4@X3dvJR>Z7GlmmY@{z1yOhSOoGH_1y)xK#)*mXNFu78
zl_4qhf|1Nc6P~sgENn(dF0ba^!Iqvl+U3=tRC<K|{p7@)&{uV@`c*|vh#2ksk^ayf
z;0YvAQ~?h=r3U6w!0HX9-g6pvb#u{!S7^sRBpO|axne9QcK056lhMD7ioCT?ISxc?
z;K)#}c0w9QFyQ(OmP8B2EM&r1PQec5xSwYIQlKVqM~|oV23=un{wf*O4s+fA%AO=^
zkfJS3aG6h@T6?xmI6w*wcU;Oh9){@4XDr1n!nz7Plc4S{{M(Ip#y>mXHnG!Bv|8Dp
z-P~guw$pOJE*+0PD=EJ%Gia5-F<C!-m9Wek0y|u7_cS5uiJUG==X>C?Pxy^#cj=)i
zO$UA$DZ8u3nkTUHvO!?qN5rddQB3t@>QjVOYn-Y<X0vYuI9RnyB^dkZ@vWQuzEvyL
zZZ&$K53vT|TAs~T-%sGBt?HLx_vU<4+n)Rj3Ae0TAEd7MN|M~fQzbS@%f!l!L5@dk
zAZcDRzkF&%B)%})Q|#a@I!rO@##akwoVvhfxo~kdN!^lK8aRF9im1k<958!NtXF8W
zSdRYf*pu_9efx#T-oS)Iw{AWbi1*Dn|2AskSwDPRAN>#`ZOJDYK#3r#ulJ+LarO%d
z>cx{=Ss)75t`I=E#aJ!dk6hf&ig;+C`5yh?EHi~+(D11$VtN>DF!Z>yxV<lWF-f&_
zrBzqe)^&s6elfRSffUy1SJOiaTE+;v57I&ND0>Vq0!rj|9MAila!^4rPE+uxZ&09X
zO8IgmLhlK;n~Of!Dz@RmTUYo>{Z1>zNL=#B>PmtTZ-JS4cM15WLd~u5=irtgMc0%-
zrlWRGLb%Gf;hl;*XR4(wN}}`KYb#P-O@gz~riZxa6@6<APejU`w7%JSEhqS$(|wF*
zxwWVi>NQj_ChlntL?E#-o;!H^o@jlNcXo`8SILXGCM1HMxF=R4N~*>10;*Q(47X9e
zrubC=fFp!I@Irh8gd-S$;VJ;fa9{1MhHj|cR-H+Nl>j%;ixA}!j0piwaRVAs=U=e*
zFP{JPY3PH+N_g%=1PA|p*`#;2_o4A%7gXHg%B0qn>>joeT{JYO>v2v-58L?vgXV9=
z*gm@PUB35o;c8LMQaEr+uq!c_vQMU}K#5#X;>b4(-CrM;UvHOk@Hw~OrN5PGr`_^K
zz<ZzBwQgu=1{(Sb9r;0{W;<g&O*>pH=5^M>sBh4Nbd^(hKzZr}&~bcc=hjSNaO4a;
zu+yTgW?5}eX7*k^a0A5EY|VVyp#?4Iv~CwEc@8N1bh%pFVd7~bL%c9`Q&|#`UCbSl
znR{(i7o-Kh`C`J~ryNRjyeH0kPn>Zo)(`9?co%FJ6w3oSPKldYH-}N57$ticjGu6%
zs&~QV-pS7*sZ8Zq?+W||YV)n(3UNF@QShP1raXK{Z5aidEo7y!13NwY4b{8(JuOp3
z=KEa-%s(dNKzPiu_L`w2W0*#nA4ZQyw;^XMBH1uVY&7y^%mgwoFv_3e8rO7KsAZG(
z>Q?MSNiE!0jnLX4@5*`-5%+=?%9_BEwx7OxnyFlxO`^EX_x$*0rn1Vf=J&=l$)7U3
z3I3WoF+hRFRGlVhZLg(YAHpXU*WYMyZYzzK7laQvN>;R6l86*<IBVuynZAd_=9}~&
z@7!yZ|4f0Rbbin5cn2&Y<}va;OUD#^(nRo{`RAVLhP3MoaGp$Czg*+N4<sq6$$q!#
zNU>g1uMxj@j{gireqjdrTH8q<-2SNQs=_a$x9coEa#8q%=S+&OUk_qDI~`%Zk92)M
z8MW&nwE0T6^dyAFI6W&sUM^ftLp3XOdwY0*XqRi4XEt8*POgZ}FT2T+Z6sYivO<yn
zzBM2uqle&2v8S3pRbdslV;{XciF_G_H_D9Bv;>cD5yHSGH$O(b&B8HL)Z;IC=insw
zZ=yi)(<nnpLywSKB&En_-YqI;D0DAo7$b8xFjA2$;<Mi0qk<N%{|XiV3XLDM`+ukD
zaAmdI9`~!v6+a2u9&#Hg{rLv}qs(q+{a3K}`MlAkh=yvrGP7A&!fPorlw7+1R6r32
zx71ox!?|tI@dsbs^HVYKEh@86IWT}_mh%MjkD_8}b>NPLSNk;GaAziIfQ(<<RWf$*
zfD#4jBy|HA-<ag>>D?0B!!;a+hH%LcZ`_*KHdUyIvlaH{Ar)+~?{{HQjlul3tbA_z
zIBMP?3g^i#@!B1`^fB@o@G!4jwbspy!!LdG`x2=yHJ*1*#Nma&G}8@r;f4!h*w?Yd
zG6hxnseABP@@3bc#i*L|o9l+@+A7fox!R>^@PLKQqTc{sO1<N0YLKgXv9e%rO@qAA
zLcw-yys6n-VK#dO`gU3QXQ2O{8n~_8CcPD9|Nf{`L#+SM4~V|sXr;r;6oq`%i$g)Q
zcC%Z49T|95xN>JY{>Q<Hm)%FqPzOZc{w{9v@i;A%m!)~X5cvFSX7$n$vHd|^nUSzT
zb}yAamP_Uw)ZOmAbmS)ySWwMw!|v!na2xDfhg8L&C|_HKFt)h;1MetkNw7+L8vkp^
zJ`p#krF`}l(gI5CP|;sbQ;%C;+?v`=#ZOJHsbAH-syyk2k`%?3JPIqtm9W0j($mQ&
zqn<$`G`&XA4yU_C%(CRGI-5dXf}68W$DWG%uvXE&#mV0y|Dzb28qg5zI&ZnKNautd
zw{6gX3r~(%#p5}fL!0M_V?CQTHM=j?Z5fa|6tq0o@4<w4`coBKCn3$=2|*!>H9Q=f
z`)}moIfF0kwUj^xxQ%8nWz^$R+%x$3<jW_Q@^>(Aip~f<Jf2Lw+kc7vmNmCK*|CPz
zWnZGt)Gh|?q_pmRUQQqIffCzap@&1S<ErNLD58sDIT+JNBr-X}J4`;An)4O9P%H0f
zAIUMKYjSPDHCQQ@1n#+-K#G|<FTVHT1t@WQ&8Z3~U<&+b<S6fSq~X`M`F6KOw%cDJ
zf@r9v<fCQ`8DQ9m9L#|mX0}_^@H<B@T1yP=%oQ6z{5-iBb~F*j_E#vWb}^8hG`R=y
zyI6gRct9b)-C=kR7#b9pNbUV#;~EeUfW1@s(9_aiyCYpkYjN@S14(R1aj{qnNPBg<
zH8@O#A4X_)R}<y*0&aY9AE9}+6{=c3vpwUtK`KvM6O@%Hz3)n3*r_(=muE*Jyykx$
zI$g6ih3{Hv^p%S1yQ{J=$7)<-CAC4%*;Q$3`7R9qah&Vx+r?ilQIEg#`ENAqBkzLM
zqXDT9V#4<-o`eg3!sG*%Ktrk0YgJT8cE1wm3SIyu65$(W6U|LyC)l0Fd9;ZVvB&@k
ziYWZTb-K(z>_mmUVGPr+%|LL9b@Ue@Y=i#BqVxV}RYk6$HmT(bwUX$a!n2v7J-d8Z
z79`|T^+wU=Tv$m4es+9KmJ~*ow5U4x!<BzO({pjRjj22mu*>X)uZT=f*Q>8wPY+A1
zQh$c=suy7xPCrIR_ah0cwNNtHQcx4AY3COAc-{0%<ZFqc=z`_<Pn^vqX4l))net__
zNO?B!G#{cmScIQ-BCiWvHIZ-tNuF`=CTw5nC|wEpQK&SL&2+cJcQT2bt(hub2hoAY
z6GwprL^C=-im^b_x9&-*@W%RpZ<z+aHd5tvI<GH<@b2K!0~H^ciSks5d<>Qv`ufsy
zCgQ1Sz((Xmn_DnjY!-`G@_f7<|3RihnazjM23I~pvw@-)G+@$xEyG2ki3_~h1ywg7
zY3*LG$psQ!Kj11}`20}Ju~sb~L64w`t2@@m!qYvm-r2Y$TeJ|ah}?g&IT<(MY~iOa
zJFD1QT>MT-qM*~mpSkO`J93ac@~rBI;qDSMxMp`bLb&WlmC63tp?l&f`gBqAb*I#t
zs#6~Z60RdtT{4wq>+Z_e@hnN{`b^5Fs8+-dpNg1h*Tr^sJdJ5=;cB^l2}n`RWZ|<I
z=Q#g*iBT0Tj)k<{QMtA(Z7Sh9TF)DJf1!3y+%PK;(s39u%fu2My##T`gH6YYG(^b(
z6Tyz=c%Z2C7Y}%0t^VGOP<<2C095EN%T2e>HFY8vzae?YbGo{@);iX&2-76Fgd13t
z)jGqwUY{3(2P*6X0@)K96fKQb&C45E$}O+W&FywX&d}?~+Qf2q@%bLR$W456FRUOJ
zxZt1y%~hor4;g(eZuMQm95jdPJ-GySZ$TDM%1a5JpX{N-xnW~<G{~hdeAoz_o{0o0
zL^T$_(n<Nk^L-3Ad!)#wn(b3$Q?=Av6fm}4H#tTnV51nWT40|l#t6@PXSF!TF<zMp
z5!hud=zoJp3_@CJh^!RyIyLZkh=7Ze%hhD^us*$?Cz;Apw2$26V2);@&wdBZv7*>c
z1<@<ztqzk7vvSr7qFiSRmPQCn!++SGvX7+Exq!D=ELC7mhw%~rokl;YyLF5+*1~Vu
zYo<qI8R*}L$>O7k+P8P4aDCw3VbqDr?081%0^1-2vlyaGwJBQ8u>^W{*~NH?tb>2H
z?B<UM)&(t3@h#SCQU0bHVi(^Z0N?-s=s)}hb;C;-Y&b3r@=&Yc6w$Fx^JK2p4(*Dq
zA<^wW(9%6h5v{53NaE@QFXx9WxtHSE!yJ*4?jwIjVSJBZX)#mLTU1aW&hJMBYK@V<
zu`5Rm{zE6$uf{dchb_<+*ErBFXy<lBp^BepsH88R{^s{wvh?q8sWy?JT@Adg_tFxs
z>+gy3R!!zc1S?I2cPA2MXT2fvnfiw)n^5Lb1nZ*)?}9fZyu5a&wF3<d#l_I&t=w`7
zoHJwWC-2NIeOcJZZ!ru1GxyK_UJ=G^Px0NKgBl4U$8^|#?UuE?`-hWK<(gBVyIB^y
z4B~vj)Gnp2+@dqDT6FQQIES)Xf6~}%J=^duo?*Y+wIxTHx7XeA9;iP!=wXbyGSsQC
z+J~R7=~&pUot}Q*>t&_$ZnXwEZ#<k)8fEq~e|y=Y9m|Ws&%r6So<+#j*B{W%W!HUH
zDUX9$URM{_Bsl6s$N~|U;!m@qC*DArN~`R6&Pw4Kqd5?2bs=EV?=CLn9@)H~T4Pqt
z@ERV1Qp+{ifJuNE%83Z^t6eL5=etD=NTG^ys1&nmR6g1?7oxx?x~slQs=;{CMH!?0
zfC=+FG5kDrVlQJXXOyL};3<w8me>SLs&gQP)I@EEt)U*R9cr~1Dlz~Cm=yLw6~2ka
z9gM^Sk&Ts?p(8O~Ou&SlOl=THEj|99;QuG{{}(SC-}Ah^Igq#WhP}OUw|3v>jrOoJ
zW@nKabbf)Dd7!2%5OWtOPYa-dO7Q=A(>xb#TlEN2T2`j<=dHT4dZqVdOVwG;05)?=
z%QIWK;rK|EAJ$SeB@KqXD>nDRCjZ=!5nPg!a)MgS2s(dqqpjkGY4mqMjcuM27J0eY
zmv3($Z=@_|2|+#<UDi2yT2P`7u`ymgCP~_?hjr!DJqvHHN5F6!E0iNGe|7jecSj|j
z6>wK*2dcj86KcAklbGK^D_v%Q4`=_otXwo?3R$Q~?^51u@%>iWc8nq2pFWq`K6C#E
zB86-3iFbMjG|wr9Yz<dnN4T_blfklO3A12DE={2@#BqpRUEIq4Jf!Uh>zOl=atd#F
zK%9n{xgKNNGeFI&Rd+%i6&0{S&|RdKlkd@G${{z$zx;~xRuGyLJ6CJDP%A1kIr6k^
z$~UtC8Y`H|94v8PZaA(IUT&U|W%_jH_8?6M3Aa9ykyX3e9{!*&$!(avqlk5(r{1GG
zct3z7s@ST0zqnKlJ$UXe<#BdK?vrpsGQ?=&XhTTz#mpz=((wtc*p@_F!{DF+-8@#F
zn)5LI<x<ucpVy03-UB`a+A)S5U3Bj`evt^R+{efwx7e=_XTH-sM2{s?^%J2D7o1uw
zoNQCTCH+^euaeZrr%-Uiw7?;1G^Vp$iTL!B<l<SXf9>k~6JPF3Ych0sa&o$5*6d!6
z@B;CPUYsY*6*Sk&5V#F*EAWq{A?n{Y1}PTyBsi+vjdCh=6wO;8bzo@KIfaggg8XPc
zqL_r+#-ReCCUWAc-2I{92f?~1xTKS?hMqvH@F&Lyaer$l<eOo8FrWg@AVcr6g7Zc3
zM{1v;<R)BjXCQl=HNj_sVr1-uOD|@kBm_b5C~sQTTtj7*tcEEoQ^$lZ+F?o48G6A2
zIjyDKbvKol_IkR^C;M{Ncc@WFAW58j5co%pT5>3X00oMA|3y<ZWgB13sQ)`)x-$;=
Qlh7XIC38gHMTaN<1|nVN)&Kwi

literal 0
HcmV?d00001

diff --git a/word2vec/paraconvert.py b/word2vec/paraconvert.py
new file mode 100755
index 0000000..5234123
--- /dev/null
+++ b/word2vec/paraconvert.py
@@ -0,0 +1,151 @@
+#!/bin/env python
+# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Example:
+    python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM
+    python paraconvert.py --t2b -i INPUT -o OUTPUT
+
+Options:
+    -h, --help  show this help message and exit
+    --b2t       convert parameter file of embedding model from binary to text
+    --t2b       convert parameter file of embedding model from text to binary
+    -i INPUT    input parameter file name
+    -o OUTPUT   output parameter file name
+    -d DIM      dimension of parameter
+"""
+from optparse import OptionParser
+import struct
+
+def binary2text(input, output, paraDim):
+    """
+    Convert a binary parameter file of embedding model to be a text file.  
+    input: the name of input binary parameter file, the format is:
+           1) the first 16 bytes is filehead:
+                version(4 bytes): version of paddle, default = 0
+                floatSize(4 bytes): sizeof(float) = 4
+                paraCount(8 bytes): total number of parameter
+           2) the next (paraCount * 4) bytes is parameters, each has 4 bytes 
+    output: the name of output text parameter file, for example:
+           0,4,32156096
+           -0.7845433,1.1937413,-0.1704215,...
+           0.0000909,0.0009465,-0.0008813,...
+           ...
+           the format is:
+           1) the first line is filehead: 
+              version=0, floatSize=4, paraCount=32156096
+           2) other lines print the paramters
+              a) each line prints paraDim paramters splitted by ','
+              b) there is paraCount/paraDim lines (embedding words)
+    paraDim: dimension of parameters 
+    """
+    fi = open(input, "rb")
+    fo = open(output, "w")
+    """
+    """
+    version, floatSize, paraCount = struct.unpack("iil", fi.read(16))
+    newHead = ','.join([str(version), str(floatSize), str(paraCount)])
+    print >> fo, newHead
+
+    bytes = 4 * int(paraDim)
+    format = "%df" % int(paraDim)
+    context = fi.read(bytes)
+    line = 0
+
+    while context:
+        numbers = struct.unpack(format, context)
+        lst = []
+        for i in numbers:
+            lst.append('%8.7f' % i)
+        print >> fo, ','.join(lst)
+        context = fi.read(bytes)
+        line += 1
+    fi.close()
+    fo.close()
+    print "binary2text finish, total", line, "lines"
+
+def get_para_count(input):
+    """
+    Compute the total number of embedding parameters in input text file. 
+    input: the name of input text file
+    """
+    numRows = 1 
+    paraDim = 0
+    with open(input) as f:
+        line = f.readline()
+        paraDim = len(line.split(","))
+        for line in f:
+            numRows += 1
+    return numRows * paraDim
+
+def text2binary(input, output, paddle_head=True):
+    """
+    Convert a text parameter file of embedding model to be a binary file.
+    input: the name of input text parameter file, for example:
+           -0.7845433,1.1937413,-0.1704215,...
+           0.0000909,0.0009465,-0.0008813,... 
+           ...
+           the format is:
+           1) it doesn't have filehead
+           2) each line stores the same dimension of parameters, 
+              the separator is commas ','
+    output: the name of output binary parameter file, the format is:
+           1) the first 16 bytes is filehead: 
+             version(4 bytes), floatSize(4 bytes), paraCount(8 bytes)
+           2) the next (paraCount * 4) bytes is parameters, each has 4 bytes
+    """
+    fi = open(input, "r")
+    fo = open(output, "wb")
+
+    newHead = struct.pack("iil", 0, 4, get_para_count(input))
+    fo.write(newHead)
+
+    count = 0
+    for line in fi:
+        line = line.strip().split(",")
+        for i in range(0, len(line)):
+            binary_data = struct.pack("f", float(line[i]))
+            fo.write(binary_data)
+        count += 1
+    fi.close()
+    fo.close()
+    print "text2binary finish, total", count, "lines"
+
+def main():
+    """
+    Main entry for running paraconvert.py 
+    """
+    usage = "usage: \n" \
+            "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \
+            "python %prog --t2b -i INPUT -o OUTPUT"
+    parser = OptionParser(usage)
+    parser.add_option("--b2t", action="store_true",
+                      help="convert parameter file of embedding model from binary to text")
+    parser.add_option("--t2b", action="store_true",
+                      help="convert parameter file of embedding model from text to binary")
+    parser.add_option("-i", action="store", dest="input",
+                      help="input parameter file name")
+    parser.add_option("-o", action="store", dest="output",
+                      help="output parameter file name")
+    parser.add_option("-d", action="store", dest="dim",
+                      help="dimension of parameter")
+    (options, args) = parser.parse_args()
+    if options.b2t:
+        binary2text(options.input, options.output, options.dim)
+    if options.t2b:
+        text2binary(options.input, options.output)
+
+if __name__ == '__main__':
+    main()
diff --git a/word2vec/train.sh b/word2vec/train.sh
new file mode 100644
index 0000000..3ddc482
--- /dev/null
+++ b/word2vec/train.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+paddle train \
+       --config Ngram.py \
+       --use_gpu=1 \
+       --dot_period=100 \
+       --log_period=3000 \
+       --test_period=0 \
+       --save_dir=model \
+       --num_passes=30
-- 
GitLab