Fix begginers' guides

Add new book's content to basics; fix the index files in basics, quick_start, and beginners' guides

source/beginners_guide/basics/03.image_classification/.gitignore

+*.pyc
+train.log
+output
+data/cifar-10-batches-py/
+data/cifar-10-python.tar.gz
+data/*.txt
+data/*.list
+data/mean.meta

source/beginners_guide/basics/03.image_classification/resnet.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle.v2 as paddle
+
+__all__ = ['resnet_cifar10']
+
+
+def conv_bn_layer(input,
+                  ch_out,
+                  filter_size,
+                  stride,
+                  padding,
+                  active_type=paddle.activation.Relu(),
+                  ch_in=None):
+    tmp = paddle.layer.img_conv(
+        input=input,
+        filter_size=filter_size,
+        num_channels=ch_in,
+        num_filters=ch_out,
+        stride=stride,
+        padding=padding,
+        act=paddle.activation.Linear(),
+        bias_attr=False)
+    return paddle.layer.batch_norm(input=tmp, act=active_type)
+
+
+def shortcut(ipt, ch_in, ch_out, stride):
+    if ch_in != ch_out:
+        return conv_bn_layer(ipt, ch_out, 1, stride, 0,
+                             paddle.activation.Linear())
+    else:
+        return ipt
+
+
+def basicblock(ipt, ch_in, ch_out, stride):
+    tmp = conv_bn_layer(ipt, ch_out, 3, stride, 1)
+    tmp = conv_bn_layer(tmp, ch_out, 3, 1, 1, paddle.activation.Linear())
+    short = shortcut(ipt, ch_in, ch_out, stride)
+    return paddle.layer.addto(input=[tmp, short], act=paddle.activation.Relu())
+
+
+def layer_warp(block_func, ipt, ch_in, ch_out, count, stride):
+    tmp = block_func(ipt, ch_in, ch_out, stride)
+    for i in range(1, count):
+        tmp = block_func(tmp, ch_out, ch_out, 1)
+    return tmp
+
+
+def resnet_cifar10(ipt, depth=32):
+    # depth should be one of 20, 32, 44, 56, 110, 1202
+    assert (depth - 2) % 6 == 0
+    n = (depth - 2) / 6
+    nStages = {16, 64, 128}
+    conv1 = conv_bn_layer(
+        ipt, ch_in=3, ch_out=16, filter_size=3, stride=1, padding=1)
+    res1 = layer_warp(basicblock, conv1, 16, 16, n, 1)
+    res2 = layer_warp(basicblock, res1, 16, 32, n, 2)
+    res3 = layer_warp(basicblock, res2, 32, 64, n, 2)
+    pool = paddle.layer.img_pool(
+        input=res3, pool_size=8, stride=1, pool_type=paddle.pooling.Avg())
+    return pool

source/beginners_guide/basics/03.image_classification/train.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import sys, os
+
+import paddle.v2 as paddle
+
+from vgg import vgg_bn_drop
+from resnet import resnet_cifar10
+
+with_gpu = os.getenv('WITH_GPU', '0') != '0'
+
+
+def main():
+    datadim = 3 * 32 * 32
+    classdim = 10
+
+    # PaddlePaddle init
+    paddle.init(use_gpu=with_gpu, trainer_count=1)
+
+    image = paddle.layer.data(
+        name="image", type=paddle.data_type.dense_vector(datadim))
+
+    # Add neural network config
+    # option 1. resnet
+    # net = resnet_cifar10(image, depth=32)
+    # option 2. vgg
+    net = vgg_bn_drop(image)
+
+    out = paddle.layer.fc(
+        input=net, size=classdim, act=paddle.activation.Softmax())
+
+    lbl = paddle.layer.data(
+        name="label", type=paddle.data_type.integer_value(classdim))
+    cost = paddle.layer.classification_cost(input=out, label=lbl)
+
+    # Create parameters
+    parameters = paddle.parameters.create(cost)
+
+    # Create optimizer
+    momentum_optimizer = paddle.optimizer.Momentum(
+        momentum=0.9,
+        regularization=paddle.optimizer.L2Regularization(rate=0.0002 * 128),
+        learning_rate=0.1 / 128.0,
+        learning_rate_decay_a=0.1,
+        learning_rate_decay_b=50000 * 100,
+        learning_rate_schedule='discexp')
+
+    # Create trainer
+    trainer = paddle.trainer.SGD(
+        cost=cost, parameters=parameters, update_equation=momentum_optimizer)
+
+    # End batch and end pass event handler
+    def event_handler(event):
+        if isinstance(event, paddle.event.EndIteration):
+            if event.batch_id % 100 == 0:
+                print "\nPass %d, Batch %d, Cost %f, %s" % (
+                    event.pass_id, event.batch_id, event.cost, event.metrics)
+            else:
+                sys.stdout.write('.')
+                sys.stdout.flush()
+        if isinstance(event, paddle.event.EndPass):
+            # save parameters
+            with open('params_pass_%d.tar' % event.pass_id, 'w') as f:
+                trainer.save_parameter_to_tar(f)
+
+            result = trainer.test(
+                reader=paddle.batch(
+                    paddle.dataset.cifar.test10(), batch_size=128),
+                feeding={'image': 0,
+                         'label': 1})
+            print "\nTest with Pass %d, %s" % (event.pass_id, result.metrics)
+
+    # Save the inference topology to protobuf.
+    inference_topology = paddle.topology.Topology(layers=out)
+    with open("inference_topology.pkl", 'wb') as f:
+        inference_topology.serialize_for_inference(f)
+
+    trainer.train(
+        reader=paddle.batch(
+            paddle.reader.shuffle(
+                paddle.dataset.cifar.train10(), buf_size=50000),
+            batch_size=128),
+        num_passes=200,
+        event_handler=event_handler,
+        feeding={'image': 0,
+                 'label': 1})
+
+    # inference
+    from PIL import Image
+    import numpy as np
+    import os
+
+    def load_image(file):
+        im = Image.open(file)
+        im = im.resize((32, 32), Image.ANTIALIAS)
+        im = np.array(im).astype(np.float32)
+        # The storage order of the loaded image is W(widht),
+        # H(height), C(channel). PaddlePaddle requires
+        # the CHW order, so transpose them.
+        im = im.transpose((2, 0, 1))  # CHW
+        # In the training phase, the channel order of CIFAR
+        # image is B(Blue), G(green), R(Red). But PIL open
+        # image in RGB mode. It must swap the channel order.
+        im = im[(2, 1, 0), :, :]  # BGR
+        im = im.flatten()
+        im = im / 255.0
+        return im
+
+    test_data = []
+    cur_dir = os.path.dirname(os.path.realpath(__file__))
+    test_data.append((load_image(cur_dir + '/image/dog.png'), ))
+
+    # users can remove the comments and change the model name
+    # with open('params_pass_50.tar', 'r') as f:
+    #    parameters = paddle.parameters.Parameters.from_tar(f)
+
+    probs = paddle.infer(
+        output_layer=out, parameters=parameters, input=test_data)
+    lab = np.argsort(-probs)  # probs and lab are the results of one batch data
+    print "Label of image/dog.png is: %d" % lab[0][0]
+
+
+if __name__ == '__main__':
+    main()

source/beginners_guide/basics/03.image_classification/vgg.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle.v2 as paddle
+
+__all__ = ['vgg_bn_drop']
+
+
+def vgg_bn_drop(input):
+    def conv_block(ipt, num_filter, groups, dropouts, num_channels=None):
+        return paddle.networks.img_conv_group(
+            input=ipt,
+            num_channels=num_channels,
+            pool_size=2,
+            pool_stride=2,
+            conv_num_filter=[num_filter] * groups,
+            conv_filter_size=3,
+            conv_act=paddle.activation.Relu(),
+            conv_with_batchnorm=True,
+            conv_batchnorm_drop_rate=dropouts,
+            pool_type=paddle.pooling.Max())
+
+    conv1 = conv_block(input, 64, 2, [0.3, 0], 3)
+    conv2 = conv_block(conv1, 128, 2, [0.4, 0])
+    conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0])
+    conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0])
+    conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])
+
+    drop = paddle.layer.dropout(input=conv5, dropout_rate=0.5)
+    fc1 = paddle.layer.fc(input=drop, size=512, act=paddle.activation.Linear())
+    bn = paddle.layer.batch_norm(
+        input=fc1,
+        act=paddle.activation.Relu(),
+        layer_attr=paddle.attr.Extra(drop_rate=0.5))
+    fc2 = paddle.layer.fc(input=bn, size=512, act=paddle.activation.Linear())
+    return fc2

source/beginners_guide/basics/04.word2vec/.gitignore

+data/train.list
+data/test.list
+data/simple-examples*

source/beginners_guide/basics/04.word2vec/calculate_dis.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Example:
+    python calculate_dis.py DICTIONARYTXT FEATURETXT
+
+Required arguments:
+    DICTIONARYTXT    the dictionary generated in dataprovider
+    FEATURETXT       the text format word feature, one line for one word
+"""
+
+import numpy as np
+from argparse import ArgumentParser
+
+
+def load_dict(fdict):
+    words = [line.strip() for line in fdict.readlines()]
+    dictionary = dict(zip(words, xrange(len(words))))
+    return dictionary
+
+
+def load_emb(femb):
+    feaBank = []
+    flag_firstline = True
+    for line in femb:
+        if flag_firstline:
+            flag_firstline = False
+            continue
+        fea = np.array([float(x) for x in line.strip().split(',')])
+        normfea = fea * 1.0 / np.linalg.norm(fea)
+        feaBank.append(normfea)
+    return feaBank
+
+
+def calcos(id1, id2, Fea):
+    f1 = Fea[id1]
+    f2 = Fea[id2]
+    return np.dot(f1.transpose(), f2)
+
+
+def get_wordidx(w, Dict):
+    if w not in Dict:
+        print 'ERROR: %s not in the dictionary' % w
+        return -1
+    return Dict[w]
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('dict', help='dictionary file')
+    parser.add_argument('fea', help='feature file')
+    args = parser.parse_args()
+
+    with open(args.dict) as fdict:
+        word_dict = load_dict(fdict)
+
+    with open(args.fea) as ffea:
+        word_fea = load_emb(ffea)
+
+    while True:
+        w1, w2 = raw_input("please input two words: ").split()
+        w1_id = get_wordidx(w1, word_dict)
+        w2_id = get_wordidx(w2, word_dict)
+        if w1_id == -1 or w2_id == -1:
+            continue
+        print 'similarity: %s' % (calcos(w1_id, w2_id, word_fea))

source/beginners_guide/basics/04.word2vec/format_convert.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Example:
+    python format_convert.py --b2t -i INPUT -o OUTPUT -d DIM
+    python format_convert.py --t2b -i INPUT -o OUTPUT
+
+Options:
+    -h, --help  show this help message and exit
+    --b2t       convert parameter file of embedding model from binary to text
+    --t2b       convert parameter file of embedding model from text to binary
+    -i INPUT    input parameter file name
+    -o OUTPUT   output parameter file name
+    -d DIM      dimension of parameter
+"""
+from optparse import OptionParser
+import struct
+
+
+def binary2text(input, output, paraDim):
+    """
+    Convert a binary parameter file of embedding model to be a text file.
+    input: the name of input binary parameter file, the format is:
+           1) the first 16 bytes is filehead:
+                version(4 bytes): version of paddle, default = 0
+                floatSize(4 bytes): sizeof(float) = 4
+                paraCount(8 bytes): total number of parameter
+           2) the next (paraCount * 4) bytes is parameters, each has 4 bytes
+    output: the name of output text parameter file, for example:
+           0,4,32156096
+           -0.7845433,1.1937413,-0.1704215,...
+           0.0000909,0.0009465,-0.0008813,...
+           ...
+           the format is:
+           1) the first line is filehead:
+              version=0, floatSize=4, paraCount=32156096
+           2) other lines print the paramters
+              a) each line prints paraDim paramters splitted by ','
+              b) there is paraCount/paraDim lines (embedding words)
+    paraDim: dimension of parameters
+    """
+    fi = open(input, "rb")
+    fo = open(output, "w")
+    """
+    """
+    version, floatSize, paraCount = struct.unpack("iil", fi.read(16))
+    newHead = ','.join([str(version), str(floatSize), str(paraCount)])
+    print >> fo, newHead
+
+    bytes = 4 * int(paraDim)
+    format = "%df" % int(paraDim)
+    context = fi.read(bytes)
+    line = 0
+
+    while context:
+        numbers = struct.unpack(format, context)
+        lst = []
+        for i in numbers:
+            lst.append('%8.7f' % i)
+        print >> fo, ','.join(lst)
+        context = fi.read(bytes)
+        line += 1
+    fi.close()
+    fo.close()
+    print "binary2text finish, total", line, "lines"
+
+
+def get_para_count(input):
+    """
+    Compute the total number of embedding parameters in input text file.
+    input: the name of input text file
+    """
+    numRows = 1
+    paraDim = 0
+    with open(input) as f:
+        line = f.readline()
+        paraDim = len(line.split(","))
+        for line in f:
+            numRows += 1
+    return numRows * paraDim
+
+
+def text2binary(input, output, paddle_head=True):
+    """
+    Convert a text parameter file of embedding model to be a binary file.
+    input: the name of input text parameter file, for example:
+           -0.7845433,1.1937413,-0.1704215,...
+           0.0000909,0.0009465,-0.0008813,...
+           ...
+           the format is:
+           1) it doesn't have filehead
+           2) each line stores the same dimension of parameters,
+              the separator is commas ','
+    output: the name of output binary parameter file, the format is:
+           1) the first 16 bytes is filehead:
+             version(4 bytes), floatSize(4 bytes), paraCount(8 bytes)
+           2) the next (paraCount * 4) bytes is parameters, each has 4 bytes
+    """
+    fi = open(input, "r")
+    fo = open(output, "wb")
+
+    newHead = struct.pack("iil", 0, 4, get_para_count(input))
+    fo.write(newHead)
+
+    count = 0
+    for line in fi:
+        line = line.strip().split(",")
+        for i in range(0, len(line)):
+            binary_data = struct.pack("f", float(line[i]))
+            fo.write(binary_data)
+        count += 1
+    fi.close()
+    fo.close()
+    print "text2binary finish, total", count, "lines"
+
+
+def main():
+    """
+    Main entry for running format_convert.py
+    """
+    usage = "usage: \n" \
+            "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \
+            "python %prog --t2b -i INPUT -o OUTPUT"
+    parser = OptionParser(usage)
+    parser.add_option(
+        "--b2t",
+        action="store_true",
+        help="convert parameter file of embedding model from binary to text")
+    parser.add_option(
+        "--t2b",
+        action="store_true",
+        help="convert parameter file of embedding model from text to binary")
+    parser.add_option(
+        "-i", action="store", dest="input", help="input parameter file name")
+    parser.add_option(
+        "-o", action="store", dest="output", help="output parameter file name")
+    parser.add_option(
+        "-d", action="store", dest="dim", help="dimension of parameter")
+    (options, args) = parser.parse_args()
+    if options.b2t:
+        binary2text(options.input, options.output, options.dim)
+    if options.t2b:
+        text2binary(options.input, options.output)
+
+
+if __name__ == '__main__':
+    main()

source/beginners_guide/basics/04.word2vec/train.py

+import math
+import os
+
+import numpy
+import paddle.v2 as paddle
+
+with_gpu = os.getenv('WITH_GPU', '0') != '0'
+
+embsize = 32
+hiddensize = 256
+N = 5
+
+
+def wordemb(inlayer):
+    wordemb = paddle.layer.table_projection(
+        input=inlayer,
+        size=embsize,
+        param_attr=paddle.attr.Param(
+            name="_proj", initial_std=0.001, learning_rate=1, l2_rate=0))
+    return wordemb
+
+
+# save and load word dict and embedding table
+def save_dict_and_embedding(word_dict, embeddings):
+    with open("word_dict", "w") as f:
+        for key in word_dict:
+            f.write(key + " " + str(word_dict[key]) + "\n")
+    with open("embedding_table", "w") as f:
+        numpy.savetxt(f, embeddings, delimiter=',', newline='\n')
+
+
+def load_dict_and_embedding():
+    word_dict = dict()
+    with open("word_dict", "r") as f:
+        for line in f:
+            key, value = line.strip().split(" ")
+            word_dict[key] = int(value)
+
+    embeddings = numpy.loadtxt("embedding_table", delimiter=",")
+    return word_dict, embeddings
+
+
+def main():
+    paddle.init(use_gpu=with_gpu, trainer_count=1)
+    word_dict = paddle.dataset.imikolov.build_dict()
+    dict_size = len(word_dict)
+    # Every layer takes integer value of range [0, dict_size)
+    firstword = paddle.layer.data(
+        name="firstw", type=paddle.data_type.integer_value(dict_size))
+    secondword = paddle.layer.data(
+        name="secondw", type=paddle.data_type.integer_value(dict_size))
+    thirdword = paddle.layer.data(
+        name="thirdw", type=paddle.data_type.integer_value(dict_size))
+    fourthword = paddle.layer.data(
+        name="fourthw", type=paddle.data_type.integer_value(dict_size))
+    nextword = paddle.layer.data(
+        name="fifthw", type=paddle.data_type.integer_value(dict_size))
+
+    Efirst = wordemb(firstword)
+    Esecond = wordemb(secondword)
+    Ethird = wordemb(thirdword)
+    Efourth = wordemb(fourthword)
+
+    contextemb = paddle.layer.concat(input=[Efirst, Esecond, Ethird, Efourth])
+    hidden1 = paddle.layer.fc(
+        input=contextemb,
+        size=hiddensize,
+        act=paddle.activation.Sigmoid(),
+        layer_attr=paddle.attr.Extra(drop_rate=0.5),
+        bias_attr=paddle.attr.Param(learning_rate=2),
+        param_attr=paddle.attr.Param(
+            initial_std=1. / math.sqrt(embsize * 8), learning_rate=1))
+    predictword = paddle.layer.fc(
+        input=hidden1,
+        size=dict_size,
+        bias_attr=paddle.attr.Param(learning_rate=2),
+        act=paddle.activation.Softmax())
+
+    cost = paddle.layer.classification_cost(input=predictword, label=nextword)
+    parameters = paddle.parameters.create(cost)
+    adagrad = paddle.optimizer.AdaGrad(
+        learning_rate=3e-3,
+        regularization=paddle.optimizer.L2Regularization(8e-4))
+    trainer = paddle.trainer.SGD(cost, parameters, adagrad)
+
+    def event_handler(event):
+        if isinstance(event, paddle.event.EndIteration):
+            if event.batch_id % 100 == 0:
+                print "Pass %d, Batch %d, Cost %f, %s" % (
+                    event.pass_id, event.batch_id, event.cost, event.metrics)
+
+        if isinstance(event, paddle.event.EndPass):
+            result = trainer.test(
+                paddle.batch(paddle.dataset.imikolov.test(word_dict, N), 32))
+            print "Pass %d, Testing metrics %s" % (event.pass_id,
+                                                   result.metrics)
+            with open("model_%d.tar" % event.pass_id, 'w') as f:
+                trainer.save_parameter_to_tar(f)
+
+    trainer.train(
+        paddle.batch(paddle.dataset.imikolov.train(word_dict, N), 32),
+        num_passes=100,
+        event_handler=event_handler)
+
+    # save word dict and embedding table
+    embeddings = parameters.get("_proj").reshape(len(word_dict), embsize)
+    save_dict_and_embedding(word_dict, embeddings)
+
+
+if __name__ == '__main__':
+    main()

source/beginners_guide/basics/05.recommender_system/.gitignore

+.idea
+.ipynb_checkpoints

source/beginners_guide/basics/06.understand_sentiment/.gitignore

+data/aclImdb
+data/imdb
+data/pre-imdb
+data/mosesdecoder-master
+*.log
+model_output
+dataprovider_copy_1.py
+model.list
+*.pyc
+.DS_Store

source/beginners_guide/basics/07.label_semantic_roles/.gitignore

+data/train.list
+data/test.*
+data/conll05st-release.tar.gz
+data/conll05st-release
+data/predicate_dict
+data/label_dict
+data/word_dict
+data/emb
+data/feature
+output
+predict.res
+train.log