diff --git a/.travis.yml b/.travis.yml index 865e21f046b7f3ac4bc3de09c1300a0a1d0337d4..d73fd39aa7a2ee87c0e31436ffc14df2213134c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ addons: - python2.7-dev - python-numpy - python-wheel + - libboost-dev - curl - swig - graphviz diff --git a/Dockerfile b/Dockerfile index 6d5797da9e06a514e03dee52bd248478fa41813f..ad0d086d3c65b5901178aa681aa36ccc0ea0c246 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 MAINTAINER PaddlePaddle Authors ARG UBUNTU_MIRROR @@ -23,11 +23,14 @@ ENV HOME /root COPY ./paddle/scripts/docker/root/ /root/ RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ liblapack-dev liblapacke-dev && \ - apt-get install -y automake locales clang-format-3.8 swig doxygen && \ + apt-get install -y \ + git python-pip python-dev openssh-server bison \ + wget unzip tar xz-utils bzip2 gzip coreutils \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-numpy python-matplotlib gcc g++ \ + automake locales clang-format-3.8 swig doxygen cmake \ + liblapack-dev liblapacke-dev libboost-dev \ + clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y # git credential to skip password typing @@ -51,11 +54,12 @@ RUN pip install --upgrade pip && \ RUN apt-get install -y libssl-dev libffi-dev RUN pip install certifi urllib3[secure] -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -VOLUME ["/woboq_out"] +# Install woboq_codebrowser to /woboq +RUN git clone https://github.com/woboq/woboq_codebrowser /woboq && \ + (cd /woboq \ + cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ + -DCMAKE_BUILD_TYPE=Release . \ + make) # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd diff --git a/RELEASE.cn.md b/RELEASE.cn.md new file mode 100755 index 0000000000000000000000000000000000000000..5deaf230a8f5dd3089993f0fc79b9460fd049750 --- /dev/null +++ b/RELEASE.cn.md @@ -0,0 +1,80 @@ +# v0.10.0版本 + +我们非常高兴发布了PaddlePaddle V0.10.0版，并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 + +- 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，分别是定义数据生成器和定义网络拓扑结构的文件。用户通过运行`paddle_trainer`的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的文件，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 + +- 新版的API被称为 *V2 API*，允许我们在单个.py文件中，通过编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用。 + +基于新的API，我们提供了一个在线的学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 + +我们还致力于迭代更新新版API的在线文档，并将新版API引入分布式集群（包括MPI和Kubernetes）训练中。我们将在下一个版本中发布更多的内容。 + +## 新特点 + +* 发布新版[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +* 发布深度学习系列课程 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 +* 支持矩形输入的CNN。 +* 为seqlastin和seqfirstin提供stride pooling。 +* 在`trainer_config_helpers`中暴露`seq_concat_layer/seq_reshape_layer`。 +* 添加公共数据集包：CIFAR，MNIST，IMDB，WMT14，CONLL05，movielens，imikolov。 +* 针对Single Shot Multibox Detection增加 Prior box layer。 +* 增加光滑的L1损失。 +* 在V2 API中增加 data reader 创建器和修饰器。 +* 增加cmrnorm投影的CPU实现。 + + +## 改进 + +* 提供`paddle_trainer`的Python virtualenv支持。 +* 增加代码自动格式化的pre-commit hooks。 +* 升级protobuf到3.x版本。 +* 在Python数据生成器中提供一个检测数据类型的选项。 +* 加速GPU中average层的后向反馈计算。 +* 细化文档。 +* 使用Travis-CI检查文档中的死链接。 +* 增加解释`sparse_vector`的示例。 +* 在layer_math.py中添加ReLU。 +* 简化Quick Start示例中的数据处理流程。 +* 支持CUDNN Deconv。 +* 在v2 API中增加数据feeder。 +* 在情感分析示例的演示中增加对标准输入流中样本的预测。 +* 提供图像预处理的多进程接口。 +* 增加V1 API的基准文档。 +* 在`layer_math.py`中增加ReLU。 +* 提供公共数据集的自动下载包。 +* 将`Argument::sumCost`重新命名为`Argument::sum`，并暴露给python。 +* 为矩阵相关的表达式评估增加一个新的`TensorExpression`实现。 +* 增加延迟分配来优化批处理多表达式计算。 +* 增加抽象的类函数及其实现： + * `PadFunc` 和 `PadGradFunc`。 + * `ContextProjectionForwardFunc` 和 `ContextProjectionBackwardFunc`。 + * `CosSimBackward` 和 `CosSimBackwardFunc`。 + * `CrossMapNormalFunc` 和 `CrossMapNormalGradFunc`。 + * `MulFunc`。 +* 增加`AutoCompare`和`FunctionCompare`类，使得编写比较gpu和cpu版本函数的单元测试更容易。 +* 生成`libpaddle_test_main.a`并删除测试文件内的主函数。 +* 支持PyDataProvider2中numpy的稠密向量。 +* 清理代码库，删除一些复制粘贴的代码片段： + * 增加`SparseRowMatrix`的抽样类`RowBuffer`。 + * 清理`GradientMachine`的接口。 + * 在layer中增加`override`关键字。 + * 简化`Evaluator::create`，使用`ClassRegister`来创建`Evaluator`。 +* 下载演示的数据集时检查MD5校验。 +* 添加`paddle::Error`，用于替代Paddle中的`LOG(FATAL)`。 + + +## 错误修复 + +* 检查`recurrent_group`的layer输入类型。 +* 不要用.cu源文件运行`clang-format`。 +* 修复`LogActivation`的使用错误。 +* 修复运行`test_layerHelpers`多次的错误。 +* 修复seq2seq示例超出消息大小限制的错误。 +* 修复在GPU模式下dataprovider转换的错误。 +* 修复`GatedRecurrentLayer`中的错误。 +* 修复在测试多个模型时`BatchNorm`的错误。 +* 修复paramRelu在单元测试时崩溃的错误。 +* 修复`CpuSparseMatrix`编译时相关的警告。 +* 修复`MultiGradientMachine`在`trainer_count > batch_size`时的错误。 +* 修复`PyDataProvider2`阻止异步加载数据的错误。 diff --git a/demo/mnist/light_mnist.py b/demo/mnist/light_mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..33409054357d2f0c6a765b3ab3164eb2e584467e --- /dev/null +++ b/demo/mnist/light_mnist.py @@ -0,0 +1,79 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +is_predict = get_config_arg("is_predict", bool, False) + +####################Data Configuration ################## + +if not is_predict: + data_dir = './data/' + define_py_data_sources2( + train_list=data_dir + 'train.list', + test_list=data_dir + 'test.list', + module='mnist_provider', + obj='process') + +######################Algorithm Configuration ############# +settings(batch_size=50, learning_rate=0.001, learning_method=AdamOptimizer()) + +#######################Network Configuration ############# + +data_size = 1 * 28 * 28 +label_size = 10 +img = data_layer(name='pixel', size=data_size) + + +# light cnn +# A shallower cnn model: [CNN, BN, ReLU, Max-Pooling] x4 + FC x1 +# Easier to train for mnist dataset and quite efficient +# Final performance is close to deeper ones on tasks such as digital and character classification +def light_cnn(input_image, num_channels, num_classes): + def __light__(ipt, + num_filter=128, + times=1, + conv_filter_size=3, + dropouts=0, + num_channels_=None): + return img_conv_group( + input=ipt, + num_channels=num_channels_, + pool_size=2, + pool_stride=2, + conv_padding=0, + conv_num_filter=[num_filter] * times, + conv_filter_size=conv_filter_size, + conv_act=ReluActivation(), + conv_with_batchnorm=True, + conv_batchnorm_drop_rate=dropouts, + pool_type=MaxPooling()) + + tmp = __light__(input_image, num_filter=128, num_channels_=num_channels) + tmp = __light__(tmp, num_filter=128) + tmp = __light__(tmp, num_filter=128) + tmp = __light__(tmp, num_filter=128, conv_filter_size=1) + + tmp = fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation()) + return tmp + + +predict = light_cnn(input_image=img, num_channels=1, num_classes=label_size) + +if not is_predict: + lbl = data_layer(name="label", size=label_size) + inputs(img, lbl) + outputs(classification_cost(input=predict, label=lbl)) +else: + outputs(predict) diff --git a/doc/api/v1/data_provider/pydataprovider2_en.rst b/doc/api/v1/data_provider/pydataprovider2_en.rst index 30357be32538db4423ad0eaf899138256c84edc7..e8fb6292779790765154502bff319ea10ab1e70b 100644 --- a/doc/api/v1/data_provider/pydataprovider2_en.rst +++ b/doc/api/v1/data_provider/pydataprovider2_en.rst @@ -178,7 +178,7 @@ input_types +++++++++++ PaddlePaddle has four data types, and three sequence types. -The four data types are: +The four data types are: * :code:`dense_vector`: dense float vector. * :code:`sparse_binary_vector`: sparse binary vector, most of the value is 0, and @@ -231,7 +231,7 @@ Its parameters lists as follows: * :code:`is_train` is a bool parameter that indicates the DataProvider is used in training or testing. * :code:`file_list` is the list of all files. - + * User-defined parameters args can be set in training configuration. Note, PaddlePaddle reserves the right to add pre-defined parameter, so please diff --git a/doc/design/cluster_train/README.md b/doc/design/cluster_train/README.md index b88a8f382bfdbbf113d2bb99e12caaa09f9cd8c5..74961f80050c6b2723889b51416a2e8048174b00 100644 --- a/doc/design/cluster_train/README.md +++ b/doc/design/cluster_train/README.md @@ -15,7 +15,7 @@ This poses technical challenges to PaddlePaddle: A training job will be created once user asks Paddle cloud to train a model. The training job is made up of different processes that collaboratively consume data and produce a trained model. There are three kinds of processes: -1. the *master process*, which dispatches tasks to +1. the *master server process*, which dispatches tasks to 1. one or more *trainer processes*, which run distributed training and synchronize gradients/models via 1. one or more *parameter server processes*, where each holds a shard of the global model, and receive the uploaded gradients from every *trainer process*, so they can run the optimize functions to update their parameters. @@ -27,9 +27,9 @@ By coordinating these processes, PaddlePaddle supports use both Synchronize Stoc When training with sync SGD, parameter servers wait for all trainers to finish gradients update and then send the updated parameters to trainers, training can not proceed until the trainer received the updated parameters. This creates a synchronization point between trainers. When training with async SGD, each trainer upload gradient and download new parameters individually, without the synchronization with other trainers. Using asyc SGD will be faster in terms of time per pass, but have more noise in gradient since trainers are likely to have a stale model. -### Master Process +### Master Server Process -The master process will: +The master server process will: - Partition a dataset into [tasks](#task) and dispatch tasks to trainers. - Keep track of training progress on the dataset with [task queue](#task-queue). A training job will iterate on the dataset for a full pass until it goes into next pass. @@ -41,11 +41,11 @@ A task is a data shard to be trained. The total number of tasks will be much big #### Task Queue -The master process has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master process. Each master process has three task queues. +The master server has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master server. Each master server process has three task queues. -- The todo queue holds tasks to be dispatched. When a job starts, the master process fills in the todo queue with all tasks. +- The todo queue holds tasks to be dispatched. When a job starts, the master server fills in the todo queue with all tasks. - The pending queue holds tasks that are currently training by trainers. - the done queue holds tasks that are already trained. @@ -54,10 +54,10 @@ The life cycle of a single task is illustrated below: 1. When a new pass of training starts, all tasks will be placed in the todo queue. -1. The master process will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. -1. The trainer will work on its tasks and tell the master process once a task is completed. The master process will dispatch a new task to that trainer. -1. If a task timeout. the master process will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. -1. The master process will move completed task to the done queue. When the todo queue is empty, the master process will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. +1. The master server will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. +1. The trainer will work on its tasks and tell the master server once a task is completed. The master server will dispatch a new task to that trainer. +1. If a task timeout. the master server will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. +1. The master server will move completed task to the done queue. When the todo queue is empty, the master server will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. ### Trainer Process @@ -93,7 +93,7 @@ The communication pattern between the trainers and the parameter servers depends ## Fault Tolerant -The training job will pause if the master processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). +The training job will pause if the master server processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). The training job will continue to make progress if there is at least one training process running. The strategy depends on the type of optimization algorithm: @@ -113,7 +113,7 @@ Now we will introduce how each process recovers from a failure, the graph below -### Master Process +### Master Server Process When the master is started by the Kubernetes, it executes the following steps at startup: @@ -122,7 +122,7 @@ When the master is started by the Kubernetes, it executes the following steps at 1. Watches the trainer prefix keys `/trainer/` on etcd to find the live trainers. 1. Starts dispatching the tasks to the trainers, and updates task queue using an etcd transaction to ensure lock is held during the update. -When the master process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. +When the master server process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. ### Trainer Process @@ -132,7 +132,7 @@ When the trainer is started by the Kubernetes, it executes the following steps a 1. Generates a unique ID, and sets key `/trainer/` with its contact address as value. The key will be deleted when the lease expires, so the master will be aware of the trainer being online and offline. 1. Waits for tasks from the master to start training. -If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master process can discover the trainer again. +If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master server can discover the trainer again. When a trainer fails, Kuberentes would try to restart it. The recovered trainer would fetch tasks from the TODO queue and go on training. diff --git a/doc/design/cluster_train/data_dispatch.md b/doc/design/cluster_train/data_dispatch.md index a3eb4e28db0782bbf88520d25023cf031e620a63..1f5d22ff5e6abcb576d16cbe7391da1967a1ab8e 100644 --- a/doc/design/cluster_train/data_dispatch.md +++ b/doc/design/cluster_train/data_dispatch.md @@ -1,30 +1,37 @@ ## 训练数据的存储和分发 +### 概念解释 + ### 流程介绍 生产环境中的训练数据集通常体积很大，并被存储在诸如Hadoop HDFS，Ceph，AWS S3之类的分布式存储之上。这些分布式存储服务通常会把数据切割成多个分片分布式的存储在多个节点之上。这样就可以在云端执行多种数据类计算任务，包括： * 数据预处理任务 * Paddle训练任务 * 在线模型预测服务 +
+ +
- - -在上图中显示了在一个实际生产环境中的应用（人脸识别）的数据流图。生产环境的日志数据会通过实时流的方式（Kafka）和离线数据的方式（HDFS）存储，并在集群中运行多个分布式数据处理任务，比如流式数据处理（online data process），离线批处理（offline data process）完成数据的预处理，提供给paddle作为训练数据。用于也可以上传labeled data到分布式存储补充训练数据。在paddle之上运行的深度学习训练输出的模型会提供给在线人脸识别的应用使用。 - -### 训练数据的存储 +在上图中显示了在一个实际生产环境中的应用（人脸识别）的数据流图。生产环境的日志数据会通过实时流的方式（Kafka）和离线数据的方式（HDFS）存储，并在集群中运行多个分布式数据处理任务，比如流式数据处理（online data process），离线批处理（offline data process）完成数据的预处理，提供给paddle作为训练数据。用户也可以上传labeled data到分布式存储补充训练数据。在paddle之上运行的深度学习训练输出的模型会提供给在线人脸识别的应用使用。 -选择CephFS作为训练数据的存储服务。 +### 训练数据存储 +我们选择[CephFS](http://docs.ceph.com/docs/master/cephfs/)作为存储系统。 -在Kubernetes上运行的不同的计算框架，可以通过Volume或PersistentVolume挂载存储空间到每个容器中。 +- 无论是从[PFSClient](../file_manager/README.md)的角度，还是从[Pod](https://kubernetes.io/docs/concepts/workloads/pods/pod/)中运行任务的角度，统一用`/pfs/$DATACENTER/home/$USER`来访问用户自己的数据。 +- `/pfs/$DATACENTER/common`下存放公共数据集合 + - 做只读挂载 -在CephFS存储系统中的公开目录，需要保存一些预置的公开数据集（比如MNIST, BOW, ImageNet数据集等），并且可以被提交的job直接使用。 +
+ +
### 文件预处理 -在数据集可以被训练之前，文件需要预先被转换成PaddlePaddle集群内部的存储格式（SSTable）。我们提供两个转换方式： -- 提供给用户本地转换的库，用户可以编写程序完成转换。 -- 用户可以上传自己的数据集，在集群运行MapReduce job完成转换。 +在开始训练之前, 数据集需要预先被转换成PaddlePaddle分布式训练使用的存储格[RecordIO](https://github.com/PaddlePaddle/Paddle/issues/1947)。我们提供两个转换方式： + +1. 用户在本地转换好再上传 +1. 用户上传数据后，在机群上运行转换程序 转换生成的文件名会是以下格式： @@ -92,11 +99,12 @@ random_images-00099-of-00099 #### 进行训练 -PaddlePaddle提供专用的[data reader creator](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md#python-data-reader-design-doc)，生成给定SSTable文件对应的data reader。**无论在本地还是在云端，reader的使用方式都是一致的**： + +PaddlePaddle提供专用的[data reader creator](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md#python-data-reader-design-doc)，生成给定`RecordIO`文件对应的data reader。**无论在本地还是在云端，reader的使用方式都是一致的**： ```python # ... -reader = paddle.reader.creator.SSTable("/home/random_images-*-of-*") +reader = paddle.reader.creator.RecordIO("/pfs/datacenter_name/home/user_name/random_images-*-of-*") batch_reader = paddle.batch(paddle.dataset.mnist.train(), 128) trainer.train(batch_reader, ...) ``` @@ -107,14 +115,46 @@ trainer.train(batch_reader, ...) 使用下面命令，可以把本地的数据上传到存储集群中。 -```bash -paddle cp filenames pfs://home/folder/ +```bash +paddle pfs cp filename /pfs/$DATACENTER/home/$USER/folder/ ``` 比如，把之前示例中转换完毕的random_images数据集上传到云端的`/home/`可以用以下指令： -```bash -paddle cp random_images-*-of-* pfs://home/ + +```bash +paddle pfs cp random_images-*-of-* /pfs/$DATACENTER/home/$USER/folder/ +``` + +需要`$DATACENTER`的配置写到配置文件中，例如 + +``` +# config file +[datacenter_1] +username=user +usercert=user.pem +userkey=user-key.pem +endpoint=datacenter1.paddlepaddle.org + +[datacenter_2] +username=user +usercert=user.pem +userkey=user-key.pem +endpoint=datacenter2.paddlepaddle.org ``` ## TODO +### 文件访问的权限 +控制用户权限 + +- 用户可以把自己的数据分享给别人 + +### 文件访问方式 +不用mount的方式来访问数据，而是直接用API的接口远程访问 + +例如： + +``` +f = open('/pfs/datacenter_name/home/user_name/test1.dat') +``` + ### 支持用户自定义的数据预处理job diff --git a/doc/design/cluster_train/master_server.md b/doc/design/cluster_train/master_server.md new file mode 100644 index 0000000000000000000000000000000000000000..bb8307652587b4dc56cd668a3a5e64722734d194 --- /dev/null +++ b/doc/design/cluster_train/master_server.md @@ -0,0 +1,91 @@ +# Design Doc: Master Server + +For an overview of master server's role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master server in more details. The master will be implemented in [Go](https://golang.org/). + +## Dataset + + + +A dataset is a list of files in *RecordIO* format. A RecordIO file consists of chunks, whereas each chunk consists some records. + +## Task Queue + +As mentioned in [distributed training design doc](./README.md), a *task* is a data shard that the master server assigns to the trainer process to train on. A task consists of one or multiple *blocks* from one or multiple files. The master server maintains *task queues* to track the training progress. + +### Task Queue Creation + +1. Each trainer will make an RPC call (using Go's [rpc](https://golang.org/pkg/net/rpc/) package) to the master server, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master server the same dataset, only the first RPC call will be honored. + + The RPC interface is: + ```go + func (m *RPCServer) ReportDataset(Paths []string, dummy *int) error { + } + ``` +1. The master server will scan through each RecordIO file to generate the *block index* and know how many blocks does each file have. A block can be referenced by the file path and the index of the block within the file. The block index is in memory data structure that enables fast access to each block, and the index of the block with the file is an integer start from 0, representing the n-th block within the file. + + The definition of the block is: + ```go + type Block struct { + Idx int // index of the block within the file + Path string + Index recordio.Index // block index + } + ``` +1. Blocks are grouped into tasks, and tasks are filled into the todo queue. The pending queue and the done queue are initialized with no element. + + The definition of the task is: + ```go + type Task struct { + Index int + Blocks []Block + } + ``` + + The elements in the tasks queues is of type `TaskEntry`, containing a timeout counter (described in [task retry logic](#task-retry-logic)), and a task: + ```go + type TaskEntry struct { + NumTimeout int + Task Task + } + ``` + + The definition of task queues is: + ```go + type TaskQueues struct { + Todo []TaskEntry + Pending map[int]TaskEntry // map from task index to task entry + Done []TaskEntry + } + ``` + +### Task Queue Persistence + +The task queues need to be persisted on [etcd](https://github.com/coreos/etcd) for fault recovery. Since the task queues only change once a task is completed or timed out, which is not very frequent, we can afford to synchronize with etcd every time the task queues change. + +We will serialize the task queues data structure with [gob encoding](https://golang.org/pkg/encoding/gob/), compress with gzip, and save into etcd synchronously under key `/task_queues`. + +### Task Dispatch + +The trainer will make an RPC call to master to get a new task when: + +- the trainer first started, or +- the trainer finishes a task. + +The RPC interface is: +```go +func (m *RPCServer) GetTask(finished *Task, result *Task) error { +} +``` +Argument `finished` will be `nil` when the trainer is just started. + +During the RPC call the master will do the following: + +- Make a copy of the task queues, and update the copy reflecting the finished tasks and the new pending tasks. +- Synchronize the copy of task queues with etcd using a transaction conditioned on holding the master lock. +- Replace the task queues with the copy and report to the trainer with the new tasks if succeeded, or discard the copy and report the error to the trainer if failed. + +### Task Retry Logic + +When a task is dispatched to the trainer, the master will schedule a function for execution after the timeout duration (based on the moving average of task completion time). If the task entry in still in the pending queue, its timeout counter will increase by one, and the task will be moved to todo queue. If the timeout counter is above the threshold, the master will log the error and discard the task. + +Please note that since a timed out task could be completed after it has been dispatched for retry, so it is possible for a task to be processed multiple times. We do not try to prevent it from happening since it's fine to train on the same task multiple times due to the stochastic nature of the stochastic gradient decent algorithm. diff --git a/doc/design/cluster_train/pserver_client.md b/doc/design/cluster_train/pserver_client.md new file mode 100644 index 0000000000000000000000000000000000000000..392bab25e9de6bf5aa7cc1b0ad345ef12f1d9e5d --- /dev/null +++ b/doc/design/cluster_train/pserver_client.md @@ -0,0 +1,157 @@ +# Design Doc: The Client Library of Parameter Server + +For an overview of trainer's role, please refer to [distributed training design doc](README.md). In this design doc, we will discuss the parameter server's client library, which will manage communication with parameter servers. The library will be implemented in [Go](https://golang.org/) and made available as a static or dynamic library with a C header file. + +## Parameter Partition + +Each parameter will be partitioned into parameter blocks to make the parameters evenly distributed on parameter servers. The partition is done automatically by the client library. The *sparse parameter* require a little different treatment: + +### Sparse Parameter + +The sparse parameter is a parameter that is updated sparsely. The name is somewhat misleading, it does not have a sparse representation, it has the same representation as a dense vector. + +Because a sparse parameter is updated sparsely, the trainer will have to partition the sparse parameter. Because the parameter server will merge all sparse parameter shard into the same file when saving the parameter. It needs special naming convention: + +If a sparse parameter is partitioned into n shards, they should be named as: + +```text +name:sparse-0 +name:sparse-1 +... +name:sparse-n-1 +``` + +The library is unaware of the partition, and treat each parameter independently. Only when saving parameters, the parameter servers will merge the sparse parameters according to the naming convention. + +## Model Optimization Using Gradients + +There are two ways to perform model optimization using gradients: + +- On Client + + The client does multiple steps of forward and backward update. In each step, the gradients are calculated and a new model is generated. After some steps, the client will calculate the difference between the newest model and the old model at step 0. The difference will be updated to parameter servers. Parameter servers will just update parameters using the difference without any optimization using gradients (such as Adam and L1 regularization). + +- On Parameter Server + + The client will send accumulated gradients to parameter servers, the parameter server will do the optimization using gradients. + +## L1 and L2 Regularization + +PaddlePaddle allows L1 or L2 regularizations to be specified per parameter, so when the trainer initializes the parameter it needs include a parameter configuration when L1 or L2 regularization is necessary. + +## Parameter Initialization + +The parameters on parameter servers need to be initialized. To provide maximum flexibility, the trainer will initialize the parameters. Only one trainer will do the initialization, the other trainers will wait for the completion of initialization and get the parameters from the parameter servers. + +### Trainer Selection + +To select the trainer for initialization, every trainer will try to get a distributed lock, whoever owns the lock will do the initialization. As illustrated below: + + + +### Trainer Selection Process + +The trainer select process is encapsulated in the C API function: +```c +int paddle_begin_init_params(paddle_pserver_client* client, const char* config_proto); +``` +The selected trainer's call to `paddle_begin_init_params` will return with 1, and the other trainers' call to `paddle_begin_init_params` will block until initialization is done, and return 0. As illustrated below: + + + +## C Interface + +```c +typedef enum { + PADDLE_ELEMENT_TYPE_INT32 = 0, + PADDLE_ELEMENT_TYPE_UINT32 = 1, + PADDLE_ELEMENT_TYPE_INT64 = 2, + PADDLE_ELEMENT_TYPE_UINT64 = 3, + PADDLE_ELEMENT_TYPE_FLOAT32 = 4, + PADDLE_ELEMENT_TYPE_FLOAT64 = 5, +} paddle_element_type; + +typedef struct { + char* name; + paddle_element_type element_type; + void* content; + int content_len; +} paddle_parameter, paddle_gradient; + +typedef struct paddle_pserver_client paddle_pserver_client; + +paddle_pserver_client* paddle_new_pserver_client(); +void paddle_pserver_client_release(paddle_pserver_client* client); + +/** + * @brief paddle_begin_init_params begins to initialize parameters on + * parameter servers. + * + * paddle_begin_init_params will be called from multiple trainers, + * only one trainer will be selected to initialize the parameters on + * parameter servers. Other trainers will be blocked until the + * initialization is done, and they need to get the initialized + * parameters from parameter servers using @paddle_get_params. + * + * @param pserver_config_proto serialized parameter server configuration in + * Protocol Buffers format. + * @return 1 if the trainer is selected to initialize parameter + * servers, otherwise 0. + */ +int paddle_begin_init_params(paddle_pserver_client* client, const char* pserver_config_proto); + +/** + * @brief paddle_init_param initializes the parameter on parameter + * servers. + * + * @param param the parameter to initialize. + * @param param_config_proto the configuration for the parameter. + * @return 0 if successful, otherwise -1. On failure, the trainer + * needs to restart the entire initialization process (starting from + * @paddle_begin_init_param). Or simply exit the program and wait for + * the cluster management system to restart the trainer. + */ +int paddle_init_param(paddle_pserver_client* client, paddle_parameter params, const char* param_config_proto); + +/** + * @brief paddle_finish_init_params tells parameter servers client has + * sent all parameters to parameter servers as initialization. + * + * @return 0 if successful, otherwise -1. On failure, the trainer + * needs to restart the entire initialization process (starting from + * @paddle_begin_init_param). Or simply exit the program and wait for + * the cluster management system to restart the trainer. + */ +int paddle_finish_init_params(paddle_pserver_client* client); + +/** + * @brief paddle_send_grads sends gradients to parameter servers for + * updating parameters. + * + * @param grads the array of gradients to send. + * @param len the length of the gradient array. + * @param learning_rate the learning rate for the gradients. + * @return 0 if successful, otherwise -1. + */ +int paddle_send_grads(paddle_pserver_client* client, const paddle_gradient* grads, int len); + +/** + * @brief paddle_get_params gets parameters from parameter servers. + * + * @param names the array of names of the parameters to get. + * @param dst the destination array of parameters to save to. + * @param len the length of the names array and the paddle_parameter + * array. + * @return 0 if successful, otherwise -1. + */ +int paddle_get_params(paddle_pserver_client* client, const char** names, paddle_parameter* dst, int len); + +/** + * @brief paddle_save_model indicates parameters to save the parameter + * to the given path + * + * @param path the path to save parameters. + * @return 0 if successful, otherwise -1. + */ +int paddle_save_model(paddle_pserver_client* client, const char* path); +``` diff --git a/doc/design/cluster_train/src/dataset.graffle b/doc/design/cluster_train/src/dataset.graffle new file mode 100644 index 0000000000000000000000000000000000000000..c10a423ed16a23229a9ee33d11bfc82bb59646c8 Binary files /dev/null and b/doc/design/cluster_train/src/dataset.graffle differ diff --git a/doc/design/cluster_train/src/dataset.png b/doc/design/cluster_train/src/dataset.png new file mode 100644 index 0000000000000000000000000000000000000000..2fb7f1cce3b6dd21489392557826e95a9f207c34 Binary files /dev/null and b/doc/design/cluster_train/src/dataset.png differ diff --git a/doc/design/cluster_train/src/file_storage.graffle b/doc/design/cluster_train/src/file_storage.graffle new file mode 100644 index 0000000000000000000000000000000000000000..50a17e70fa255495337c529a3bf12a5c0024a5be Binary files /dev/null and b/doc/design/cluster_train/src/file_storage.graffle differ diff --git a/doc/design/cluster_train/src/file_storage.png b/doc/design/cluster_train/src/file_storage.png new file mode 100644 index 0000000000000000000000000000000000000000..fccb4e3e7e738224c7f1584326bd5f351ce799aa Binary files /dev/null and b/doc/design/cluster_train/src/file_storage.png differ diff --git a/doc/design/cluster_train/src/init_lock.graffle b/doc/design/cluster_train/src/init_lock.graffle new file mode 100644 index 0000000000000000000000000000000000000000..fa9149f21b1311eed48ef72ec55e556559d0fc94 Binary files /dev/null and b/doc/design/cluster_train/src/init_lock.graffle differ diff --git a/doc/design/cluster_train/src/init_lock.png b/doc/design/cluster_train/src/init_lock.png new file mode 100644 index 0000000000000000000000000000000000000000..92404ee6d6c0f9a7727952bae3c869ba338ecd7f Binary files /dev/null and b/doc/design/cluster_train/src/init_lock.png differ diff --git a/doc/design/cluster_train/src/pserver_init.graffle b/doc/design/cluster_train/src/pserver_init.graffle new file mode 100644 index 0000000000000000000000000000000000000000..730d3a561ffdc19e723b3cf6612471440951826a Binary files /dev/null and b/doc/design/cluster_train/src/pserver_init.graffle differ diff --git a/doc/design/cluster_train/src/pserver_init.png b/doc/design/cluster_train/src/pserver_init.png new file mode 100644 index 0000000000000000000000000000000000000000..4d502226d82ba271c50ae1bec5efaaaac4cc4434 Binary files /dev/null and b/doc/design/cluster_train/src/pserver_init.png differ diff --git a/doc/design/cluster_train/src/submit-job.graffle b/doc/design/cluster_train/src/submit-job.graffle new file mode 100644 index 0000000000000000000000000000000000000000..677cdfb6d9a32168bf71729eb841fa1ca0dd31d6 Binary files /dev/null and b/doc/design/cluster_train/src/submit-job.graffle differ diff --git a/doc/design/cluster_train/src/submit-job.png b/doc/design/cluster_train/src/submit-job.png new file mode 100644 index 0000000000000000000000000000000000000000..3046a460a7ba708079e88a560debaa215a694680 Binary files /dev/null and b/doc/design/cluster_train/src/submit-job.png differ diff --git a/doc/design/cluster_train/submit-job.md b/doc/design/cluster_train/submit-job.md new file mode 100644 index 0000000000000000000000000000000000000000..8377d5489dc64bd2fdc5bb4f7bc737e7b489000d --- /dev/null +++ b/doc/design/cluster_train/submit-job.md @@ -0,0 +1,127 @@ +# Submit a Distributed Training Job + +The user can submit a distributed training job with Python code, rather than with a command-line interface. + +## Runtime Environment On Kubernetes + +For a distributed training job, there is two Docker image called *runtime Docker image* and *base Docker image*. The runtime Docker image is the Docker image that gets scheduled by Kubernetes to run during training. The base Docker image is for building the runtime Docker image. + +### Base Docker Image + +Usually, the base Docker image is PaddlePaddle product Docker image including paddle binary files and python package. And of course, users can specify any image name hosted on any docker registry which users have the access right. + +### Runtime Docker Image + +The trainer package which user upload and some Python dependencies are packaged into a runtime Docker image based on base Docker image. + +- Handle Python Dependencies + + You need to provide requirements.txt file in your `trainer-package` folder. Example: + + ```txt + pillow + protobuf==3.1.0 + ``` + More [details](https://pip.readthedocs.io/en/1.1/requirements.html) about requirements, an example project looks like: + ```bash + paddle_example + |-quick_start + |-trainer.py + |-dataset.py + |-requirements.txt + ``` + +## Submit Distributed Training Job With Python Code + + +- `paddle.job.dist_train()` will call the Job Server API `/v1/packages` to upload the trainer package and save them on CephFS, and then call `/v1/trainer/job` to submit the PaddlePaddle distributed job. +- `/v1/trainer/job` will start a building job for preparing the runtime Docker image. When the building job is finished, Job Server will submit the PaddlePaddle distributed job to Kubernetes. +- *NOTE*: For the first version, we will not prepare the runtime Docker image, instead, the package is uploaded to Paddle Cloud, and Paddle Cloud will mount the package in a temporary folder into the base Docker image. We will not support custom Python dependencies in the first version as well. + +You can call `paddle.job.dist_train` and provide distributed training configuration as the parameters: +```python +paddle.job.dist_train( + trainer=dist_trainer(), + paddle_job=PaddleJob( + job_name = "paddle-cloud", + entry_point = "python %s"%__file__, + trainer_package = "/example/word2vec", + image = "yancey1989/paddle-job", + trainers = 10, + pservers = 3, + trainer_cpu = 1, + trainer_gpu = 1, + trainer_mem = "10G", + pserver_cpu = 1, + pserver_mem = "2G" + )) +``` + +The parameter `trainer` of `paddle.job.dist_train` is a function and you can implement it as follows: +```python +def dist_trainer(): + def trainer_creator(): + trainer = paddle.v2.trainer.SGD(...) + trainer.train(...) + return trainer_creator +``` + +The pseudo code of `paddle.job.dist_train` is as follows: +```python +def dist_train(trainer, paddle_job): + # if the code is running on cloud, set PADDLE_ON_CLOUD=YES + if os.getenv("RUNNING_ON_CLOUD", "NO") == "NO": + #submit the paddle job + paddle_job.submit() + else: + #start the training + trainer() +``` +### PaddleJob Parameters +parameter | type | explanation + --- | --- | --- +job_name | str | the unique name for the training job +entry_point | str | entry point for startup trainer process +trainer_package | str | trainer package file path which user have the access right +image|str|the [base image](#base-docker-image) for building the [runtime image](#runtime-docker-image) +pservers|int| Parameter Server process count +trainers|int| Trainer process count +pserver_cpu|int| CPU count for each Parameter Server process +pserver_mem|str| memory allocated for each Parameter Server process, a plain integer using one of these suffixes: E, P, T, G, M, K +trainer_cpu|int| CPU count for each Trainer process +trainer_mem|str| memory allocated for each Trainer process, a plain integer using one of these suffixes: E, P, T, G, M, K +trainer_gpu|int| GPU count for each Trainer process, if you only want CPU, do not set this parameter + +### Deploy Parameter Server, Trainer and Master Process + - Deploy PaddlePaddle Parameter Server processes, it's a Kubernetes ReplicaSet. + - Deploy PaddlePaddle Trainer processes, it's a Kubernetes Job. + - Deploy PaddlePaddle Master processes, it's a Kubernetes ReplicaSet. + +## Job Server + +- RESTful API + + Job server provides RESTful HTTP API for receiving the trainer package and displaying + PaddlePaddle job related informations. + - `POST /v1/package` receive the trainer package and save them on CephFS + - `POST /v1/trainer/job` submit a trainer job + - `GET /v1/jobs/` list all jobs + - `GET /v1/jobs/` the status of a job + - `DELETE /v1/jobs/` delete a job + - `GET /v1/version` job server version + +- Build Runtime Docker Image on Kubernetes + + `paddle.job.dist_train` will upload the trainer package to Job Server, save them on the distributed filesystem, and then start up a job for building the runtime Docker image that gets scheduled by Kubernetes to run during training. + + There are some benefits for building runtime Docker image on JobServer: + - On Paddle Cloud, users will run the trainer code in a Jupyter Notebook which is a Kubernetes Pod, if we want to execute `docker build` in the Pod, we should mount the host's `docker.sock` to the Pod, user's code will connect the host's Docker Engine directly, it's not safe. + - Users only need to upload the training package files, does not need to install docker engine, docker registry as dependencies. + - If we want to change another image type, such as RKT, users do not need to care about it. + +- Deploy Parameter Server, Trainer and Master Processes + + `POST /v1/trainer/job` receives the distributed training parameters, and deploy the job as follows: + - Deploy PaddlePaddle Parameter Server processes, it's a Kubernetes ReplicaSet. + - Deploy PaddlePaddle Trainer processes, it's a Kubernetes Job. + - Deploy PaddlePaddle Master processes, it's a Kubernetes ReplicaSet. diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index df5e172252277a881480cd2816eb901b711abe6b..c14160d55ec8fdb9fc552da33f3a3dac13c1a764 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -232,7 +232,19 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 用户需要指定本机上Python的路径：````, ````, ```` -10. A protocol message was rejected because it was too big +11. CMake源码编译，Paddle版本号为0.0.0 +-------------------------------------- + +如果运行 :code:`paddle version`, 出现 :code:`PaddlePaddle 0.0.0`；或者运行 :code:`cmake ..`，出现 + +.. code-block:: bash + + CMake Warning at cmake/version.cmake:20 (message): + Cannot add paddle version from git tag + +那么用户需要拉取所有的远程分支到本机，命令为 :code:`git fetch upstream`，然后重新cmake即可。 + +12. A protocol message was rejected because it was too big ---------------------------------------------------------- 如果在训练NLP相关模型时，出现以下错误： @@ -270,7 +282,7 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 完整源码可参考 `seqToseq `_ 示例。 -11. 如何指定GPU设备 +13. 如何指定GPU设备 ------------------- 例如机器上有4块GPU，编号从0开始，指定使用2、3号GPU： @@ -288,7 +300,7 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 paddle train --use_gpu=true --trainer_count=2 --gpu_id=2 -12. 训练过程中出现 :code:`Floating point exception`, 训练因此退出怎么办? +14. 训练过程中出现 :code:`Floating point exception`, 训练因此退出怎么办? ------------------------------------------------------------------------ Paddle二进制在运行时捕获了浮点数异常，只要出现浮点数异常(即训练过程中出现NaN或者Inf)，立刻退出。浮点异常通常的原因是浮点数溢出、除零等问题。 diff --git a/doc/getstarted/concepts/src/train.py b/doc/getstarted/concepts/src/train.py new file mode 100644 index 0000000000000000000000000000000000000000..679d0a931a7d650108ea89a04080a55d2976f72e --- /dev/null +++ b/doc/getstarted/concepts/src/train.py @@ -0,0 +1,52 @@ +import paddle.v2 as paddle +import numpy as np + +# init paddle +paddle.init(use_gpu=False) + +# network config +x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) +y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) +y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) +cost = paddle.layer.mse_cost(input=y_predict, label=y) + +# create parameters +parameters = paddle.parameters.create(cost) +# create optimizer +optimizer = paddle.optimizer.Momentum(momentum=0) +# create trainer +trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + + +# event_handler to print training info +def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % (event.pass_id, event.batch_id, + event.cost) + + +# define training dataset reader +def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + + return reader + + +# define feeding map +feeding = {'x': 0, 'y': 1} + +# training +trainer.train( + reader=paddle.batch( + train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..e63ca11102c8ce457afcc3c262fa5f159361c01d --- /dev/null +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -0,0 +1,150 @@ +############ +基本使用概念 +############ + +PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松地完成神经网络配置，模型训练等任务。 +这里将介绍PaddlePaddle的基本使用概念，并且展示了如何利用PaddlePaddle来解决一个经典的线性回归问题。 +在使用该文档之前，请参考 `安装文档 <../build_and_install/index_cn.html>`_ 完成PaddlePaddle的安装。 + + +配置网络 +============ + +加载PaddlePaddle +---------------------- + +在进行网络配置之前，首先需要加载相应的Python库，并进行初始化操作。 + +.. code-block:: bash + + import paddle.v2 as paddle + import numpy as np + paddle.init(use_gpu=False) + + +搭建神经网络 +----------------------- + +搭建神经网络就像使用积木搭建宝塔一样。在PaddlePaddle中，layer是我们的积木，而神经网络是我们要搭建的宝塔。我们使用不同的layer进行组合，来搭建神经网络。 +宝塔的底端需要坚实的基座来支撑，同样，神经网络也需要一些特定的layer作为输入接口，来完成网络的训练。 + +例如，我们可以定义如下layer来描述神经网络的输入： + +.. code-block:: bash + + x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) + y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) + +其中x表示输入数据是一个维度为2的稠密向量，y表示输入数据是一个维度为1的稠密向量。 + +PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和三种序列模式。 + +四种数据类型： + +* dense_vector：稠密的浮点数向量。 +* sparse_binary_vector：稀疏的01向量，即大部分值为0，但有值的地方必须为1。 +* sparse_float_vector：稀疏的向量，即大部分值为0，但有值的部分可以是任何浮点数。 +* integer：整数标签。 + +三种序列模式： + +* SequenceType.NO_SEQUENCE：不是一条序列 +* SequenceType.SEQUENCE：是一条时间序列 +* SequenceType.SUB_SEQUENCE： 是一条时间序列，且序列的每一个元素还是一个时间序列。 + +不同的数据类型和序列模式返回的格式不同，列表如下： + ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| | NO_SEQUENCE | SEQUENCE | SUB_SEQUENCE | ++======================+=====================+===================================+================================================+ +| dense_vector | [f, f, ...] | [[f, ...], [f, ...], ...] | [[[f, ...], ...], [[f, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_binary_vector | [i, i, ...] | [[i, ...], [i, ...], ...] | [[[i, ...], ...], [[i, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_float_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| integer_value | i | [i, i, ...] | [[i, ...], [i, ...], ...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ + +其中，f代表一个浮点数，i代表一个整数。 + +注意：对sparse_binary_vector和sparse_float_vector，PaddlePaddle存的是有值位置的索引。例如， + +- 对一个5维非序列的稀疏01向量 ``[0, 1, 1, 0, 0]`` ，类型是sparse_binary_vector，返回的是 ``[1, 2]`` 。 +- 对一个5维非序列的稀疏浮点向量 ``[0, 0.5, 0.7, 0, 0]`` ，类型是sparse_float_vector，返回的是 ``[(1, 0.5), (2, 0.7)]`` 。 + + +在定义输入layer之后，我们可以使用其他layer进行组合。在组合时，需要指定layer的输入来源。 + +例如，我们可以定义如下的layer组合： + +.. code-block:: bash + + y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) + cost = paddle.layer.mse_cost(input=y_predict, label=y) + +其中，x与y为之前描述的输入层；而y_predict是接收x作为输入，接上一个全连接层；cost接收y_predict与y作为输入，接上均方误差层。 + +最后一层cost中记录了神经网络的所有拓扑结构，通过组合不同的layer，我们即可完成神经网络的搭建。 + + +训练模型 +============ + +在完成神经网络的搭建之后，我们首先需要根据神经网络结构来创建所需要优化的parameters，并创建optimizer。 +之后，我们可以创建trainer来对网络进行训练。 + +.. code-block:: bash + + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Momentum(momentum=0) + trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + +其中，trainer接收三个参数，包括神经网络拓扑结构、神经网络参数以及迭代方程。 + +在搭建神经网络的过程中，我们仅仅对神经网络的输入进行了描述。而trainer需要读取训练数据进行训练，PaddlePaddle中通过reader来加载数据。 + +.. code-block:: bash + + # define training dataset reader + def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + return reader + +最终我们可以调用trainer的train方法启动训练： + +.. code-block:: bash + + # define feeding map + feeding = {'x': 0, 'y': 1} + + # event_handler to print training info + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % ( + event.pass_id, event.batch_id, event.cost) + # training + trainer.train( + reader=paddle.batch(train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) + +关于PaddlePaddle的更多使用方法请参考 `进阶指南 <../../howto/index_cn.html>`_。 + +线性回归完整示例 +============== + +下面给出在三维空间中使用线性回归拟合一条直线的例子： + +.. literalinclude:: src/train.py + :linenos: + +有关线性回归的实际应用，可以参考PaddlePaddle book的 `第一章节 `_。 \ No newline at end of file diff --git a/doc/getstarted/index_cn.rst b/doc/getstarted/index_cn.rst index cadf092f8f42ca16bbeb23bd21e0d018af8e43cc..0cb27f802c40ef123fdc9c6799aad3b2a5f554c0 100644 --- a/doc/getstarted/index_cn.rst +++ b/doc/getstarted/index_cn.rst @@ -5,5 +5,6 @@ :maxdepth: 1 build_and_install/index_cn.rst + concepts/use_concepts_cn.rst - `深度学习入门课程 `_ diff --git a/doc/howto/dev/contribute_to_paddle_cn.md b/doc/howto/dev/contribute_to_paddle_cn.md index 775938612e8d213b92e2eb69dae805838dc5ae96..a48b143c760c6fc6fc08e793e4cf2f82f6713dc0 100644 --- a/doc/howto/dev/contribute_to_paddle_cn.md +++ b/doc/howto/dev/contribute_to_paddle_cn.md @@ -7,6 +7,7 @@ - 确保编译器选项 `WITH_STYLE_CHECK` 已打开，并且编译能通过代码样式检查。 - 所有代码必须具有单元测试。 - 通过所有单元测试。 +- 请遵守[提交代码的一些约定](#提交代码的一些约定)。 以下教程将指导您提交代码。 ## [Fork](https://help.github.com/articles/fork-a-repo/) @@ -217,3 +218,22 @@ upstream ``` 至此，我们就完成了一次代码贡献的过程。 + +## 提交代码的一些约定 + +为了使评审人在评审代码时更好地专注于代码本身，请您每次提交代码时，遵守以下约定： +1. 请保证Travis-CI 中单元测试能顺利通过。如果没过，说明提交的代码存在问题，评审人一般不做评审。 +2. 提交PUll Request前： + - 请注意commit的数量： + - 原因：如果仅仅修改一个文件但提交了十几个commit，每个commit只做了少量的修改，这会给评审人带来很大困扰。评审人需要逐一查看每个commit才能知道做了哪些修改，且不排除commit之间的修改存在相互覆盖的情况。 + - 建议：每次提交时，保持尽量少的commit，可以通过`git commit --amend`补充上次的commit。对已经Push到远程仓库的多个commit，可以参考[squash commits after push](http://stackoverflow.com/questions/5667884/how-to-squash-commits-in-git-after-they-have-been-pushed)。 + - 请注意每个commit的名称：应能反映当前commit的内容，不能太随意。 +3. 如果解决了某个Issue的问题，请在该PUll Request的**第一个**评论框中加上：`fix #issue_number`，这样当该PUll Request被合并后，会自动关闭对应的Issue。关键词包括：close, closes, closed, fix, fixes, fixed, resolve, resolves, resolved，请选择合适的词汇。详细可参考[Closing issues via commit messages](https://help.github.com/articles/closing-issues-via-commit-messages)。 + +此外，在回复评审人意见时，请您遵守以下约定： +1. 评审人的每个意见都必须回复（这是开源社区的基本礼貌，别人帮了忙，应该说谢谢）： + - 对评审意见同意且按其修改完的，给个简单的`Done`即可； + - 对评审意见不同意的，请给出您自己的反驳理由。 +2. 如果评审意见比较多： + - 请给出总体的修改情况。 + - 请采用[start a review](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/)进行回复，而非直接回复的方式。原因是每个回复都会发送一封邮件，会造成邮件灾难。 diff --git a/doc/howto/index_cn.rst b/doc/howto/index_cn.rst index 5b84eea491f874459ed2071e4c942657cdc9b18b..26449a6365843b526b3ac3111b337d2f17524c9d 100644 --- a/doc/howto/index_cn.rst +++ b/doc/howto/index_cn.rst @@ -8,7 +8,6 @@ :maxdepth: 1 usage/cmd_parameter/index_cn.rst - usage/concepts/use_concepts_cn.rst usage/cluster/cluster_train_cn.md usage/k8s/k8s_basis_cn.md usage/k8s/k8s_cn.md diff --git a/doc/howto/usage/concepts/src/pserver_topology.dot b/doc/howto/usage/concepts/src/pserver_topology.dot deleted file mode 100644 index 9ff658b8495030f322d4f553f3bf72ddf8d3a578..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/src/pserver_topology.dot +++ /dev/null @@ -1,68 +0,0 @@ -graph pp_topology { - rankdir=BT; - subgraph cluster_node0 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器0" - - pserver0 [label="Parameter \n Server 0"] - trainer0 [label="Trainer 0"] - } - subgraph cluster_node1 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器1" - - pserver1 [label="Parameter \n Server 1"] - trainer1 [label="Trainer 1"] - } - - subgraph cluster_node2 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器2" - - pserver2 [label="Parameter \n Server 2"] - trainer2 [label="Trainer 2"] - } - - subgraph cluster_node3 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器3" - - pserver3 [label="Parameter \n Server 3"] - trainer3 [label="Trainer 3"] - } - - data [label="数据", shape=hexagon] - - trainer0 -- pserver0 - trainer0 -- pserver1 - trainer0 -- pserver2 - trainer0 -- pserver3 - - trainer1 -- pserver0 - trainer1 -- pserver1 - trainer1 -- pserver2 - trainer1 -- pserver3 - - trainer2 -- pserver0 - trainer2 -- pserver1 - trainer2 -- pserver2 - trainer2 -- pserver3 - - trainer3 -- pserver0 - trainer3 -- pserver1 - trainer3 -- pserver2 - trainer3 -- pserver3 - - data -- trainer0 - data -- trainer1 - data -- trainer2 - data -- trainer3 -} diff --git a/doc/howto/usage/concepts/src/trainer_config.py b/doc/howto/usage/concepts/src/trainer_config.py deleted file mode 100644 index 3eccbd7bc11f4865130286de718d1be74e4d1722..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/src/trainer_config.py +++ /dev/null @@ -1,29 +0,0 @@ -from paddle.trainer_config_helpers import * - -define_py_data_sources2( - train_list='train.list', - test_list='test.list', - module='provider', - obj='process') -settings( - batch_size=128, - learning_rate=1e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(0.5)) - -img = data_layer(name='pixel', size=28 * 28) - -hidden1 = simple_img_conv_pool( - input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) - -hidden2 = fc_layer( - input=hidden1, - size=200, - act=TanhActivation(), - layer_attr=ExtraAttr(drop_rate=0.5)) -predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) - -outputs( - classification_cost( - input=predict, label=data_layer( - name='label', size=10))) diff --git a/doc/howto/usage/concepts/use_concepts_cn.rst b/doc/howto/usage/concepts/use_concepts_cn.rst deleted file mode 100644 index fa334bcbb9e29d6943def7c35fa53e1b9262d29c..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/use_concepts_cn.rst +++ /dev/null @@ -1,139 +0,0 @@ -############ -基本使用概念 -############ - -PaddlePaddle是一个深度学习框架，支持单机模式和多机模式。 - -单机模式用命令 ``paddle train`` 可以启动一个trainer进程，单机训练通常只包括一个trainer进程。如果数据规模比较大，希望加速训练，可以启动分布式作业。一个分布式作业里包括若干trainer进程和若干Parameter Server（或称pserver）进程。用命令 ``paddle pserver`` 可以启动 pserver 进程，pserver进程用于协调多个trainer进程之间的通信。 - -本文首先介绍trainer进程中的一些使用概念，然后介绍pserver进程中概念。 - -.. contents:: - -系统框图 -======== - -下图描述了用户使用框图，PaddlePaddle的trainer进程里内嵌了Python解释器，trainer进程可以利用这个解释器执行Python脚本，Python脚本里定义了模型配置、训练算法、以及数据读取函数。其中，数据读取程序往往定义在一个单独Python脚本文件里，被称为数据提供器（DataProvider），通常是一个Python函数。模型配置、训练算法通常定义在另一单独Python文件中, 称为训练配置文件。下面将分别介绍这两部分。 - -.. graphviz:: - - digraph pp_process { - rankdir=LR; - config_file [label="用户神经网络配置"]; - subgraph cluster_pp { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "PaddlePaddle C++"; - py [label="Python解释器"]; - } - data_provider [label="用户数据解析"]; - config_file -> py; - py -> data_provider [dir="back"]; - } - -数据提供器 -========== - -DataProvider是PaddlePaddle系统的数据提供器，将用户的原始数据转换成系统可以识别的数据类型。每当系统需要新的数据训练时, trainer进程会调用DataProvider函数返回数据。当所有数据读取完一轮后，DataProvider返回空数据，通知系统一轮数据读取结束，并且系统每一轮训练开始时会重置DataProvider。需要注意的是，DataProvider是被系统调用，而不是新数据驱动系统，一些随机化噪声添加都应该在DataProvider中完成。 - -在不同的应用里，训练数据的格式往往各不相同。因此，为了用户能够灵活的处理数据，我们提供了Python处理数据的接口，称为 ``PyDataProvider`` 。在 ``PyDataProvider`` 中，系统C++模块接管了shuffle、处理batch、GPU和CPU通信、双缓冲、异步读取等问题，一些情况下(如：``min_pool_size=0``)需要Python接口里处理shuffle，可以参考 :ref:`api_pydataprovider2` 继续深入了解。 - - -训练配置文件 -============ - -训练配置文件主要包括数据源、优化算法、网络结构配置三部分。 其中数据源配置与DataProvider的关系是：DataProvider里定义数据读取函数，训练配置文件的数据源配置中指定DataProvider文件名字、生成数据函数接口，请不要混淆。 - -一个简单的训练配置文件为： - -.. literalinclude:: src/trainer_config.py - :linenos: - -文件开头 ``from paddle.trainer_config_helpers import *`` ，是因为PaddlePaddle配置文件与C++模块通信的最基础协议是protobuf，为了避免用户直接写复杂的protobuf string，我们为用户定以Python接口来配置网络，该Python代码可以生成protobuf包，这就是 :ref:`api_trainer_config` 的作用。因此，在文件的开始，需要import这些函数。 这个包里面包含了模型配置需要的各个模块。 - -下面分别介绍数据源配置、优化算法配置、网络结构配置这三部分该概念。 - -数据源配置 ----------- - -使用 ``PyDataProvider2`` 的函数 ``define_py_data_sources2`` 配置数据源。``define_py_data_sources2`` 里通过train_list和test_list指定是训练文件列表和测试文件列表。 如果传入字符串的话，是指一个数据列表文件。这个数据列表文件中包含的是每一个训练或者测试文件的路径。如果传入一个list的话，则会默认生成一个list文件，再传入给train.list或者test.list。 - -``module`` 和 ``obj`` 指定了DataProvider的文件名和返回数据的函数名。更详细的使用，请参考 :ref:`api_pydataprovider2` 。 - -优化算法配置 ------------- - -通过 :ref:`api_trainer_config_helpers_optimizers_settings` 接口设置神经网络所使用的训练参数和 :ref:`api_trainer_config_helpers_optimizers` ，包括学习率、batch_size、优化算法、正则方法等，具体的使用方法请参考 :ref:`api_trainer_config_helpers_optimizers_settings` 文档。 - -网络结构配置 ------------- - -神经网络配置主要包括网络连接、激活函数、损失函数、评估器。 - -- 网络连接： 主要由Layer组成，每个Layer返回的都是一个 ``LayerOutput`` 对象，Layer里面可以定义参数属性、激活类型等。 - - 为了更灵活的配置，PaddlePaddle提供了基于 Projection 或者 Operator 的配置，这两个需要与 ``mixed_layer`` 配合使用。这里简单介绍Layer、Projection、Operator的概念: - - - Layer: 神经网络的某一层，可以有可学习的参数，一般是封装了许多复杂操作的集合。 - - Projection：需要与 ``mixed_layer`` 配合使用，含可学习参数。 - - Operator： 需要与 ``mixed_layer`` 配合使用，不含可学习参数，输入全是其他Layer的输出。 - - - 这个配置文件网络由 ``data_layer`` 、 ``simple_img_conv_pool`` 、 ``fc_layer`` 组成。 - - - :ref:`api_trainer_config_helpers_layers_data_layer` ： 通常每个配置文件都会包括 ``data_layer`` ，定义输入数据大小。 - - :ref:`api_trainer_config_helpers_network_simple_img_conv_pool` ：是一个组合层，包括了图像的卷积 (convolution)和池化(pooling)。 - - :ref:`api_trainer_config_helpers_layers_fc_layer` ：全连接层，激活函数为Softmax，这里也可叫分类层。 - -- 损失函数和评估器：损失函数即为网络的优化目标，评估器可以评价模型结果。 - - PaddlePaddle包括很多损失函数和评估起，详细可以参考 :ref:`api_trainer_config_helpers_layers_cost_layers` 和 :ref:`api_trainer_config_helpers_evaluators` 。这里 ``classification_cost`` 默认使用多类交叉熵损失函数和分类错误率统计评估器。 - -- ``outputs``: 标记网络输出的函数为 ``outputs`` 。 - - 训练阶段，网络的输出为神经网络的优化目标；预测阶段，网络的输出也可通过 ``outputs`` 标记。 - - -这里对 ``mixed_layer`` 稍做详细说明， 该Layer将多个输入(Projection 或 Operator)累加求和，具体计算是通过内部的 Projection 和 Operator 完成，然后加 Bias 和 activation 操作， - -例如，和 ``fc_layer`` 同样功能的 ``mixed_layer`` 是: - -.. code-block:: python - - data = data_layer(name='data', size=200) - with mixed_layer(size=200) as out: - out += full_matrix_projection(input=data) - -PaddlePaddle 可以使用 ``mixed layer`` 配置出非常复杂的网络，甚至可以直接配置一个完整的LSTM。用户可以参考 :ref:`api_trainer_config_helpers_layers_mixed_layer` 的相关文档进行配置。 - - -分布式训练 -========== - -PaddlePaddle多机采用经典的 Parameter Server 架构对多个节点的 trainer 进行同步。多机训练的经典拓扑结构如下\: - -.. graphviz:: src/pserver_topology.dot - -图中每个灰色方块是一台机器，在每个机器中，先使用命令 ``paddle pserver`` 启动一个pserver进程，并指定端口号，可能的参数是\: - -.. code-block:: bash - - paddle pserver --port=5000 --num_gradient_servers=4 --tcp_rdma='tcp' --nics='eth0' - -* ``--port=5000`` : 指定 pserver 进程端口是 5000 。 -* ``--gradient_servers=4`` : 有四个训练进程(PaddlePaddle 将 trainer 也称作 GradientServer ，因为其为负责提供Gradient) 。 -* ``--tcp_rdma='tcp' --nics=`eth0```: 指定以太网类型为TCP网络，指定网络接口名字为eth0。 - -启动之后 pserver 进程之后，需要启动 trainer 训练进程，在各个机器上运行如下命令\: - -.. code-block:: bash - - paddle train --port=5000 --pservers=192.168.100.101,192.168.100.102,192.168.100.103,192.168.100.104 --config=... - -对于简单的多机协同训练使用上述方式即可。另外，pserver/train 通常在高级情况下，还需要设置下面两个参数\： - -* --ports_num\: 一个 pserver 进程共绑定多少个端口用来做稠密更新，默认是1。 -* --ports_num_for_sparse\: 一个pserver进程共绑定多少端口用来做稀疏更新，默认是0。 - -使用手工指定端口数量，是因为Paddle的网络通信中，使用了 int32 作为消息长度，比较容易在大模型下溢出。所以，在 pserver 进程中可以启动多个子线程去接受 trainer 的数据，这样单个子线程的长度就不会溢出了。但是这个值不可以调的过大，因为增加这个值，对性能尤其是内存占用有一定的开销，另外稀疏更新的端口如果太大的话，很容易导致某一个参数服务器没有分配到任何参数。 diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index eff296bcb0174c71e05ee169c85236343a3e1164..c6fd9cc54ae3a671c5bdcf54cbaa873c59280694 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -9,6 +9,14 @@ add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) +find_package(boost QUIET) + +if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + add_subdirectory(majel) +endif() + if(WITH_C_API) add_subdirectory(capi) endif() diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 4d0dacae9058f94e584f313c9d0e31b5af09e82d..1cec77c0cae6ffbf7a1ca22092e8e41a6f9f0fc5 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -92,7 +92,9 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so # TODO(yuyang18) : make wheel name calculated by cmake add_custom_target(python_api_wheel ALL DEPENDS ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so) -install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ DESTINATION opt/paddle/share/wheels) +install(DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/dist/ + DESTINATION opt/paddle/share/wheels +) if(WITH_TESTING) IF(NOT PY_PIP_FOUND) diff --git a/paddle/majel/.gitignore b/paddle/majel/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1f5acdebb56971202b63d2485e2ac5042786f13c --- /dev/null +++ b/paddle/majel/.gitignore @@ -0,0 +1,2 @@ +build +third-party \ No newline at end of file diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..baa3bb9e914b3053a18dc638146325ffe3d28a12 --- /dev/null +++ b/paddle/majel/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.0) + +if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})") +else() + # find #include + get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) + include_directories(${PARENT_DIR}) + + # find cmake directory modules + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + + # enable c++11 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + + # enable gtest + set(THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/third_party) + set(WITH_TESTING ON) + include(external/gtest) +endif() + +########################### Build Majel ############################# +set(MAJEL_CXX_FILES place.cc) +set(MAJEL_CUDA_FILES "") + +if(CUDA_FOUND) + cuda_add_library(majel ${MAJEL_CUDA_FILES} ${MAJEL_CXX_FILES}) +else() + add_library(majel ${MAJEL_CXX_FILES}) +endif() +##################################################################### + +add_subdirectory(test) diff --git a/paddle/majel/README.md b/paddle/majel/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5539853056797284ca1fa5ef5ab16fa0059907f0 --- /dev/null +++ b/paddle/majel/README.md @@ -0,0 +1,126 @@ +# Tensor: An Unified Data Type in PaddlePaddle + +## Pain Point + +In this week, we discussed several potential weaknesses of PaddlePaddle caused by rapid iteration and development to promote new business products on the line in recent four years. For instance, current Matrix/Vector implementation in PaddlePaddle are long and tedious to read, which interfered seriously with the contribution of both fresh and professional engineers. More seriously for this issue, it will also become too challenging to maintain over time. + + +## Learn from Majel + +Consequently, we decide to refactor PaddlePaddle step-by-step. First, refactor and replace Matrix/Vector to Tensor, a modern terminology in the deep learning system. Fortunately, we can learn from Majel how to define a Tensor. + +To simplify heterogeneous resource allocation in any dimensions (1-9) and types (double, float, float16), Majel consists of several primitives such as `Dim`, `Place` and `Array`, all of them are standard C++ class templates. + +1. `Place`: memory location [i.e. CPU/GPU]. +2. `Allocation`: heterogeneous resource allocator [i.e. 20MB in GPU]. +3. `Dim`: size of each dimension. [i.e. Dim<4>({10, 2, 5, 1})] +4. `Array`: dynamic array consists of `Place`, `Dim`, and a pointer to memory. + +If you dig deeper into Majel source code, you will find Majel heavily use `boost.variant`. The variant class template is a safe, generic, stack-based discriminated union container, **offering a simple solution for manipulating an object from a heterogeneous set of types in a uniform manner**. Whereas standard containers such as std::vector may be thought of as "multi-value, single type," variant is "multi-type, single value." + +As a simple example, consider the following: + +```c++ +#include "boost/variant.hpp" +#include + +class my_visitor : public boost::static_visitor +{ +public: + int operator()(int i) const + { + return i; + } + + int operator()(const std::string & str) const + { + return str.length(); + } +}; + +int main() +{ + boost::variant< int, std::string > u("hello world"); + std::cout << u; // output: hello world + + int result = boost::apply_visitor( my_visitor(), u ); + std::cout << result; // output: 11 (i.e., length of "hello world") +} +``` + +In Majel, `DDimVar` is derived from `Dim`, `DArrayVar` is from `Array`. + +```c++ +template +struct Dim { +... +int head; +Dim tail; +} +``` + +```c++ +template +class Array : public Buffer { + ... +private: + Dim size_; + Dim stride_; + T* ptr_; +}; +``` + +```c++ +typedef boost::variant Place; +typedef boost::variant, Dim<2>, Dim<3>, Dim<4>, Dim<5>, + Dim<6>, Dim<7>, Dim<8>, Dim<9>> DDimVar; +typedef boost::variant< + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array > DArrayVar; +``` + +Because `variant` may be thought of as "multi-type, single value", we can utilize it to implement unified interfaces for PaddlePaddle. + +## implement Tensor in Paddle + +Before writing code, please make sure you already look through Majel Source Code and grabbed the design philosophy of `DArray` in Majel. + +To assign subtasks to our colleagues, we have to discuss how to divide it to independent subtasks. + +- [ ] 1. First, we need to consider the third-party dependencies in Majel. + + Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. + +> @gangliao + +- [ ] 2. Re-implement `Place` and `Allocation/Memory` + + I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation`, because we have the GPU development experience before joining Paddle team. + +> @wangkuiyi @gangliao @qijun + +- [ ] 3. Re-implement `Dim`. + + `Dim` is an excellent implementation in Majel. + +> ??? + +- [ ] 4. Re-implement `Array/Tensor`. + +> Prerequisites: 1 - 3 + +- [ ] 5. Re-implement fundamental operators for `Array/Tensor`. + +> Prerequisites: 1 - 4 diff --git a/paddle/majel/place.cc b/paddle/majel/place.cc new file mode 100644 index 0000000000000000000000000000000000000000..eecd8e5b730704258d2bd7d98a75a0a80e13a797 --- /dev/null +++ b/paddle/majel/place.cc @@ -0,0 +1,49 @@ +#include + +namespace majel { + +namespace detail { + +class PlacePrinter : public boost::static_visitor<> { +private: + std::ostream& os_; + +public: + PlacePrinter(std::ostream& os) : os_(os) {} + + void operator()(const CpuPlace&) { os_ << "CpuPlace"; } + + void operator()(const GpuPlace& p) { os_ << "GpuPlace(" << p.device << ")"; } +}; + +} // namespace majel + +static Place the_default_place; + +void set_place(const Place& place) { the_default_place = place; } + +const Place& get_place() { return the_default_place; } + +const GpuPlace default_gpu() { return GpuPlace(0); } + +const CpuPlace default_cpu() { return CpuPlace(); } + +bool is_gpu_place(const Place& p) { + return boost::apply_visitor(IsGpuPlace(), p); +} + +bool is_cpu_place(const Place& p) { + return !boost::apply_visitor(IsGpuPlace(), p); +} + +bool places_are_same_class(const Place& p1, const Place& p2) { + return is_gpu_place(p1) == is_gpu_place(p2); +} + +std::ostream& operator<<(std::ostream& os, const majel::Place& p) { + majel::detail::PlacePrinter printer(os); + boost::apply_visitor(printer, p); + return os; +} + +} // namespace majel diff --git a/paddle/majel/place.h b/paddle/majel/place.h new file mode 100644 index 0000000000000000000000000000000000000000..ad3dc3fe0b80ac5dc10a59910c580d7912469cd4 --- /dev/null +++ b/paddle/majel/place.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include + +namespace majel { + +struct CpuPlace { + CpuPlace() {} // WORKAROUND: for some reason, omitting this constructor + // causes errors with boost 1.59 and OSX + // needed for variant equality comparison + inline bool operator==(const CpuPlace&) const { return true; } + + inline bool operator!=(const CpuPlace&) const { return false; } +}; + +struct GpuPlace { + GpuPlace(int d) : device(d) {} + + // needed for variant equality comparison + inline bool operator==(const GpuPlace& o) const { return device == o.device; } + + inline bool operator!=(const GpuPlace& o) const { return !(*this == o); } + + GpuPlace() : GpuPlace(0) {} + int device; +}; + +class IsGpuPlace : public boost::static_visitor { +public: + bool operator()(const CpuPlace&) const { return false; } + + bool operator()(const GpuPlace& gpu) const { return true; } +}; + +typedef boost::variant Place; + +void set_place(const Place&); + +const Place& get_place(); + +const GpuPlace default_gpu(); +const CpuPlace default_cpu(); + +bool is_gpu_place(const Place&); +bool is_cpu_place(const Place&); +bool places_are_same_class(const Place&, const Place&); + +std::ostream& operator<<(std::ostream&, const majel::Place&); + +} // namespace majel diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..46da6ff89b4a1d68fe4229b4f0f051000ab390c7 --- /dev/null +++ b/paddle/majel/test/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE ALL_TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cc") + +add_executable(majel_tests ${ALL_TEST_FILES}) +add_dependencies(majel_tests majel) +target_link_libraries(majel_tests + ${Boost_LIBRARIES} + ${GTEST_LIBRARIES} + ${GTEST_MAIN_LIBRARIES} + majel + ) +add_test(majel_tests majel_tests) diff --git a/paddle/majel/test/place_test.cc b/paddle/majel/test/place_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..c9a53802b23ef8b225b9e8ef0acfe1b0c5562289 --- /dev/null +++ b/paddle/majel/test/place_test.cc @@ -0,0 +1,40 @@ +#include "majel/place.h" +#include +#include "gtest/gtest.h" + +TEST(Place, Equality) { + majel::CpuPlace cpu; + majel::GpuPlace g0(0), g1(1), gg0(0); + + EXPECT_EQ(cpu, cpu); + EXPECT_EQ(g0, g0); + EXPECT_EQ(g1, g1); + EXPECT_EQ(g0, gg0); + + EXPECT_NE(g0, g1); + + EXPECT_TRUE(majel::places_are_same_class(g0, gg0)); + EXPECT_FALSE(majel::places_are_same_class(g0, cpu)); +} + +TEST(Place, Default) { + EXPECT_TRUE(majel::is_gpu_place(majel::get_place())); + EXPECT_TRUE(majel::is_gpu_place(majel::default_gpu())); + EXPECT_TRUE(majel::is_cpu_place(majel::default_cpu())); + + majel::set_place(majel::CpuPlace()); + EXPECT_TRUE(majel::is_cpu_place(majel::get_place())); +} + +TEST(Place, Print) { + { + std::stringstream ss; + ss << majel::GpuPlace(1); + EXPECT_EQ("GpuPlace(1)", ss.str()); + } + { + std::stringstream ss; + ss << majel::CpuPlace(); + EXPECT_EQ("CpuPlace", ss.str()); + } +} diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 9e72c68d4bb1e3708b01885460b10a0f9dc5afe8..101b44e6c62ecf0b84d65ee7b6e90e64bd7b3272 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -4,9 +4,9 @@ set -xe # Set BASE_IMAGE according to env variables if [ ${WITH_GPU} == "ON" ]; then - BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu14.04" + BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04" else - BASE_IMAGE="ubuntu:14.04" + BASE_IMAGE="ubuntu:16.04" fi DOCKERFILE_GPU_ENV="" @@ -22,6 +22,20 @@ cd /paddle/build # build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true +cat < /paddle/build/Dockerfile <]" - echo "This script generates doc and doc_cn in the script's directory." - echo "These are common commands used in various situations:" - echo " with_docker build doc and doc_cn with docker" - echo " local build doc and doc_cn locally" -} - - -case "$1" in - "with_docker") - docker run --rm -v $PWD/../../../../:/paddle \ - -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_DOC=ON" paddledev/paddle:dev - ;; - "local") - mkdir -p doc - mkdir -p doc_cn - PADDLE_SOURCE_DIR=$PWD/../../../../ - mkdir -p $PADDLE_SOURCE_DIR/build_doc - pushd $PADDLE_SOURCE_DIR/build_doc - cmake .. -DWITH_DOC=ON - make paddle_docs paddle_docs_cn - popd - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/en/html/* doc - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/cn/html/* doc_cn - rm -rf $PADDLE_SOURCE_DIR/build_doc - ;; - "--help") - usage - ;; - *) - usage - ;; -esac +docker run --rm \ + -v $(git rev-parse --show-toplevel):/paddle \ + -e "WITH_GPU=OFF" \ + -e "WITH_AVX=ON" \ + -e "WITH_DOC=ON" \ + -e "WOBOQ=ON" \ + ${1:-"paddledev/paddle:dev"} diff --git a/paddle/utils/DynamicLoader.cpp b/paddle/utils/DynamicLoader.cpp index 76cf3c300616e6961be905d0e54f3b9fac4922a4..5604a90038b06d2c1a4d9db70e4185cddfd25d3e 100644 --- a/paddle/utils/DynamicLoader.cpp +++ b/paddle/utils/DynamicLoader.cpp @@ -104,20 +104,9 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, CHECK(nullptr != *dso_handle) << "Failed to find dynamic library: " << dlPath << " (" << dlerror() << ") \n" << "Please specify its path correctly using " - "one of the following ways: \n" - - << "Method 1. set cuda and cudnn lib path at " - "runtime. " - << "http://www.paddlepaddle.org/doc/ui/" - "cmd_argument/" - "argument_outline.html \n" - << "For instance, issue command: paddle train " - "--use_gpu=1 " - << "--cuda_dir=/usr/local/cuda/lib64 " - "--cudnn_dir=/usr/local/cudnn/lib " - "...\n" - - << "Method 2. set environment variable " + "following ways: \n" + + << "Method. set environment variable " "LD_LIBRARY_PATH on Linux or " << "DYLD_LIBRARY_PATH on Mac OS. \n" << "For instance, issue command: export " @@ -126,9 +115,7 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, << "Note: After Mac OS 10.11, using the " "DYLD_LIBRARY_PATH is impossible " << "unless System Integrity Protection (SIP) " - "is disabled. However, " - "method 1 " - << "always work well."; + "is disabled."; } void GetCublasDsoHandle(void** dso_handle) { diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 0e752c117c1ecfab72e2da2f830380e9524236e7..a36f0ebfdcb9f90f54ba2d688f9f4bcee2939ef3 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -270,7 +270,7 @@ class CheckWrapper(object): assert isinstance(each, collections.Sequence) for d in each: assert isinstance(d, float) - assert len(each, input_type.dim) + assert len(each) == input_type.dim elif input_type.type == DataType.Index: assert isinstance(each, int) assert each < input_type.dim @@ -304,7 +304,7 @@ class CheckInputTypeWrapper(object): def __call__(self, obj, filename): for items in self.generator(obj, filename): try: - # dict type is required for input_types when item is dict type + # dict type is required for input_types when item is dict type assert (isinstance(items, dict) and \ not isinstance(self.input_types, dict))==False yield items diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 32e31fe2c446fb5d5e2df0819749a60cb8afdfd2..57d30b088b873a94a11483aea536a9e4f6493129 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -138,14 +138,7 @@ def init_config_environment( g_root_submodel=None, g_submodel_map={}, g_submodel_stack=[], - g_add_submodel_suffix=False, - - # Whether current layer needs to pass the image height and width. - # Default value is true, but if it encounters recurrent_layer_group, - # it will be false. The reason is that image is converted to be sequence, - # image height will be sequence length, and image width will be feature - # length of each timestep. - g_pass_height_width=True, ): + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -1437,12 +1430,6 @@ class LayerBase(object): g_current_submodel.layer_names.append(self.config.name) - if self.config.type != 'data' and g_pass_height_width: - height = self.get_input_layer(0).height - width = self.get_input_layer(0).width - if height and width: - self.set_layer_height_width(height, width) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] @@ -3164,8 +3151,6 @@ class WarpCTCLayer(LayerBase): @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): def __init__(self, name, device=None): - global g_pass_height_width - g_pass_height_width = False super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 31652613fb3a55636b32babbc4bde60d65776c61..3b6f0270de16627821624dd1266a0a1c089323b0 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -3765,7 +3765,7 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() @layer_support() -def mse_cost(input, label, weight=None, name=None, layer_attr=None): +def mse_cost(input, label, weight=None, name=None, coeff=1.0, layer_attr=None): """ mean squared error cost: @@ -3782,6 +3782,8 @@ def mse_cost(input, label, weight=None, name=None, layer_attr=None): :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput + :param coeff: The coefficient affects the gradient in the backward. + :type coeff: float :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -3793,6 +3795,7 @@ def mse_cost(input, label, weight=None, name=None, layer_attr=None): inputs=ipts, type="square_error", name=name, + coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COST, parents=parents, size=1) @@ -4798,6 +4801,7 @@ def crf_layer(input, weight=None, param_attr=None, name=None, + coeff=1.0, layer_attr=None): """ A layer for calculating the cost of sequential conditional random @@ -4824,6 +4828,8 @@ def crf_layer(input, :type param_attr: ParameterAttribute :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param coeff: The coefficient affects the gradient in the backward. + :type coeff: float :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -4848,6 +4854,7 @@ def crf_layer(input, type=LayerType.CRF_LAYER, size=size, inputs=ipts, + coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input, label] if weight is not None: @@ -4921,12 +4928,14 @@ def crf_decoding_layer(input, @wrap_act_default(act=SigmoidActivation()) @wrap_bias_attr_default(has_bias=True) +@wrap_param_attr_default() @wrap_name_default() @layer_support() def nce_layer(input, label, - num_classes, + num_classes=None, act=None, + param_attr=None, weight=None, num_neg_samples=10, neg_distribution=None, @@ -4942,7 +4951,8 @@ def nce_layer(input, .. code-block:: python - cost = nce_layer(input=layer1, label=layer2, weight=layer3, + cost = nce_layer(input=[layer1, layer2], label=layer2, + param_attr=[attr1, attr2], weight=layer3, num_classes=3, neg_distribution=[0.1,0.3,0.6]) :param name: layer name @@ -4957,6 +4967,8 @@ def nce_layer(input, :type num_classes: int :param act: Activation, default is Sigmoid. :type act: BaseActivation + :param param_attr: The Parameter Attribute|list. + :type param_attr: ParameterAttribute :param num_neg_samples: number of negative samples. Default is 10. :type num_neg_samples: int :param neg_distribution: The distribution for generating the random negative labels. @@ -4972,9 +4984,20 @@ def nce_layer(input, """ if isinstance(input, LayerOutput): input = [input] + assert not isinstance(param_attr, collections.Sequence) + param_attr = [param_attr] + else: + if isinstance(param_attr, collections.Sequence): + assert len(input) == len(param_attr) + else: + param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] + assert isinstance(input, collections.Sequence) + assert isinstance(label, LayerOutput) assert label.layer_type == LayerType.DATA + if num_classes is None: + num_classes = label.size if neg_distribution is not None: assert isinstance(neg_distribution, collections.Sequence) assert len(neg_distribution) == num_classes @@ -4984,9 +5007,9 @@ def nce_layer(input, ipts_for_layer = [] parents = [] - for each_input in input: + for each_input, attr in zip(input, param_attr): assert isinstance(each_input, LayerOutput) - ipts_for_layer.append(each_input.name) + ipts_for_layer.append(Input(each_input.name, **attr.attr)) parents.append(each_input) ipts_for_layer.append(label.name) parents.append(label) @@ -5363,7 +5386,7 @@ def multi_binary_label_cross_entropy(input, @wrap_name_default() @layer_support() -def smooth_l1_cost(input, label, name=None, layer_attr=None): +def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): """ This is a L1 loss but more smooth. It requires that the size of input and label are equal. The formula is as follows, @@ -5392,6 +5415,8 @@ def smooth_l1_cost(input, label, name=None, layer_attr=None): :type input: LayerOutput :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param coeff: The coefficient affects the gradient in the backward. + :type coeff: float :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -5405,6 +5430,7 @@ def smooth_l1_cost(input, label, name=None, layer_attr=None): name=name, type=LayerType.SMOOTH_L1, inputs=[input.name, label.name], + coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput( name, LayerType.SMOOTH_L1, parents=[input, label], size=1) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr index 9fae596f281d44dc24c45cb3c750233266e95948..fd5224ca55cd1f642ca2f927f867a7cbf8a47cf6 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr @@ -90,8 +90,6 @@ layers { input_layer_name: "__pool_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 32 - width: 32 } parameters { name: "___conv_0__.w0" diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr index 05fd1c99d2db6e9faa3b3884ec9baf051791f9fe..05847344be60b4de42a7dd709914fd3da524d1ae 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr @@ -215,6 +215,22 @@ layers { } coeff: 1.0 } +layers { + name: "__nce_layer_0__" + type: "nce" + size: 1 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___nce_layer_0__.w0" + } + inputs { + input_layer_name: "labels" + } + bias_parameter_name: "___nce_layer_0__.wbias" + num_classes: 5000 + num_neg_samples: 10 +} parameters { name: "___fc_layer_0__.w0" size: 800 @@ -245,6 +261,26 @@ parameters { initial_strategy: 0 initial_smart: true } +parameters { + name: "___nce_layer_0__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.0141421356237 + dims: 5000 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___nce_layer_0__.wbias" + size: 5000 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 5000 + initial_strategy: 0 + initial_smart: false +} input_layer_names: "input" input_layer_names: "labels" input_layer_names: "crf_label" @@ -267,6 +303,7 @@ output_layer_names: "__cross_entropy_with_selfnorm_0__" output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" output_layer_names: "__sum_cost_0__" +output_layer_names: "__nce_layer_0__" sub_models { name: "root" layer_names: "input" @@ -292,6 +329,7 @@ sub_models { layer_names: "__huber_cost_0__" layer_names: "__multi_binary_label_cross_entropy_0__" layer_names: "__sum_cost_0__" + layer_names: "__nce_layer_0__" input_layer_names: "input" input_layer_names: "labels" input_layer_names: "crf_label" @@ -314,6 +352,7 @@ sub_models { output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" output_layer_names: "__sum_cost_0__" + output_layer_names: "__nce_layer_0__" is_recurrent_layer_group: false } diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr index 3244181a63109335c4fba6ca4dd04ac8f0446313..b7d74f85ab4ca3f434dfa45516dfee7227b6ceee 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr @@ -60,6 +60,31 @@ layers { } coeff: 1.0 } +layers { + name: "multi_class_label" + type: "data" + size: 500 + active_type: "" +} +layers { + name: "__nce_layer_0__" + type: "nce" + size: 1 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___nce_layer_0__.w0" + } + inputs { + input_layer_name: "multi_class_label" + } + inputs { + input_layer_name: "weight" + } + bias_parameter_name: "___nce_layer_0__.wbias" + num_classes: 500 + num_neg_samples: 10 +} parameters { name: "___fc_layer_0__.w0" size: 3000 @@ -80,9 +105,30 @@ parameters { initial_strategy: 0 initial_smart: false } +parameters { + name: "___nce_layer_0__.w0" + size: 5000 + initial_mean: 0.0 + initial_std: 0.04472135955 + dims: 500 + dims: 10 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___nce_layer_0__.wbias" + size: 500 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 500 + initial_strategy: 0 + initial_smart: false +} input_layer_names: "input" input_layer_names: "label" input_layer_names: "weight" +input_layer_names: "multi_class_label" output_layer_names: "__cost_0__" output_layer_names: "__mse_cost_0__" evaluators { @@ -100,9 +146,12 @@ sub_models { layer_names: "__fc_layer_0__" layer_names: "__cost_0__" layer_names: "__mse_cost_0__" + layer_names: "multi_class_label" + layer_names: "__nce_layer_0__" input_layer_names: "input" input_layer_names: "label" input_layer_names: "weight" + input_layer_names: "multi_class_label" output_layer_names: "__cost_0__" output_layer_names: "__mse_cost_0__" evaluator_names: "classification_error_evaluator" diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr index c763a95f9d1aefa022f38e0beef6d1c86ebb360d..03f4f3a31d6c222d949f64341bb8ac4c2a56fc5a 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr @@ -153,8 +153,6 @@ layers { img_size_y: 0 } } - height: 24 - width: 24 } layers { name: "__fc_layer_0__" @@ -165,8 +163,6 @@ layers { input_layer_name: "__block_expand_layer_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 24 - width: 24 } parameters { name: "___conv_0__.w0" diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py index 18ff6b48c495b7a9d61595916ade1a54b1fa6a10..d2a3b702a1d7b650947b344e4719098f68d4dd73 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -40,4 +40,6 @@ outputs( name='huber_label', size=1)), multi_binary_label_cross_entropy( input=probs, label=xe_label), - sum_cost(input=hidden)) + sum_cost(input=hidden), + nce_layer( + input=hidden, label=labels)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py index 1c0aa7f9b9ee45b9eaf82dc46a2648d834dcd4ad..c369062930e2b067ceab0dc3b25ba6c1eabe2450 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py @@ -11,4 +11,9 @@ outputs( classification_cost( input=fc, label=lbl, weight=wt), mse_cost( - input=fc, label=lbl, weight=wt)) + input=fc, label=lbl, weight=wt), + nce_layer( + input=fc, + label=data_layer( + name='multi_class_label', size=500), + weight=wt)) diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 70c8ec0baa63b9a20506cb786997c0b6d8eedf01..851fe7060fd52120603ebabb4069d67471aa05d0 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -33,11 +33,12 @@ import networks import py_paddle.swig_paddle as api import minibatch import plot +import image __all__ = [ 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', 'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader', - 'topology', 'networks', 'infer', 'plot', 'evaluator' + 'topology', 'networks', 'infer', 'plot', 'evaluator', 'image' ] @@ -52,7 +53,7 @@ def init(**kwargs): args_dict.update(kwargs) # NOTE: overwrite arguments from ENV if it is in kwargs for key in args_dict.keys(): - args.append('--%s=%s' % (key, str(kwargs[key]))) + args.append('--%s=%s' % (key, str(args_dict[key]))) api.initPaddle(*args) diff --git a/python/paddle/v2/dataset/imikolov.py b/python/paddle/v2/dataset/imikolov.py index bf88fe15570682b544bbac802cd65545f756fc3e..dd3a4552d2e1a2b00dde5ddb7ac1d78445bdca51 100644 --- a/python/paddle/v2/dataset/imikolov.py +++ b/python/paddle/v2/dataset/imikolov.py @@ -28,6 +28,11 @@ URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' +class DataType(object): + NGRAM = 1 + SEQ = 2 + + def word_count(f, word_freq=None): if word_freq is None: word_freq = collections.defaultdict(int) @@ -41,7 +46,7 @@ def word_count(f, word_freq=None): return word_freq -def build_dict(typo_freq=50): +def build_dict(min_word_freq=50): """ Build a word dictionary from the corpus, Keys of the dictionary are words, and values are zero-based IDs of these words. @@ -59,7 +64,7 @@ def build_dict(typo_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = filter(lambda x: x[1] > typo_freq, word_freq.items()) + word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) @@ -69,7 +74,7 @@ def build_dict(typo_freq=50): return word_idx -def reader_creator(filename, word_idx, n): +def reader_creator(filename, word_idx, n, data_type): def reader(): with tarfile.open( paddle.v2.dataset.common.download( @@ -79,16 +84,27 @@ def reader_creator(filename, word_idx, n): UNK = word_idx[''] for l in f: - l = [''] + l.strip().split() + [''] - if len(l) >= n: + if DataType.NGRAM == data_type: + assert n > -1, 'Invalid gram length' + l = [''] + l.strip().split() + [''] + if len(l) >= n: + l = [word_idx.get(w, UNK) for w in l] + for i in range(n, len(l) + 1): + yield tuple(l[i - n:i]) + elif DataType.SEQ == data_type: + l = l.strip().split() l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) + src_seq = [word_idx['']] + l + trg_seq = l + [word_idx['']] + if n > 0 and len(src_seq) > n: continue + yield src_seq, trg_seq + else: + assert False, 'Unknow data type' return reader -def train(word_idx, n): +def train(word_idx, n, data_type=DataType.NGRAM): """ imikolov training set creator. @@ -97,15 +113,18 @@ def train(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Training reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n, + data_type) -def test(word_idx, n): +def test(word_idx, n, data_type=DataType.NGRAM): """ imikolov test set creator. @@ -114,12 +133,15 @@ def test(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Test reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n, + data_type) def fetch(): diff --git a/python/paddle/v2/dataset/tests/imikolov_test.py b/python/paddle/v2/dataset/tests/imikolov_test.py index 009e55243a594e5e235c36fb0223ec70754d17f3..4e52810e6b924e0796e3d836dbbcb27ede2c9e25 100644 --- a/python/paddle/v2/dataset/tests/imikolov_test.py +++ b/python/paddle/v2/dataset/tests/imikolov_test.py @@ -13,10 +13,37 @@ class TestMikolov(unittest.TestCase): n = 5 self.check_reader(paddle.v2.dataset.imikolov.train(WORD_DICT, n), n) + first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + 'rake regatta rubens sim snack-food ssangyong swapo wachter' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.train( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_test(self): n = 5 self.check_reader(paddle.v2.dataset.imikolov.test(WORD_DICT, n), n) + first_line = 'consumers may want to move their telephones a little '\ + 'closer to the tv set' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.test( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_total(self): _, idx = zip(*WORD_DICT.items()) self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py new file mode 100644 index 0000000000000000000000000000000000000000..13f53919aa49694f722d4bf20a7d01af3e3e6533 --- /dev/null +++ b/python/paddle/v2/image.py @@ -0,0 +1,236 @@ +import numpy as np +try: + import cv2 +except ImportError: + cv2 = None + +from cv2 import resize + +__all__ = [ + "load_image", "resize_short", "to_chw", "center_crop", "random_crop", + "left_right_flip", "simple_transform", "load_and_transform" +] +""" +This file contains some common interfaces for image preprocess. +Many users are confused about the image layout. We introduce +the image layout as follows. + +- CHW Layout + - The abbreviations: C=channel, H=Height, W=Width + - The default layout of image opened by cv2 or PIL is HWC. + PaddlePaddle only supports the CHW layout. And CHW is simply + a transpose of HWC. It must transpose the input image. + +- Color format: RGB or BGR + OpenCV use BGR color format. PIL use RGB color format. Both + formats can be used for training. Noted that, the format should + be keep consistent between the training and inference peroid. +""" + + +def load_image(file, is_color=True): + """ + Load an color or gray image from the file path. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + + :param file: the input image path. + :type file: string + :param is_color: If set is_color True, it will load and + return a color image. Otherwise, it will + load and return a gray image. + """ + # cv2.IMAGE_COLOR for OpenCV3 + # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version + # cv2.IMAGE_GRAYSCALE for OpenCV3 + # cv2.CV_LOAD_IMAGE_GRAYSCALE for older OpenCV Version + # Here, use constant 1 and 0 + # 1: COLOR, 0: GRAYSCALE + flag = 1 if is_color else 0 + im = cv2.imread(file, flag) + return im + + +def resize_short(im, size): + """ + Resize an image so that the length of shorter edge is size. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the shorter edge size of image after resizing. + :type size: int + """ + assert im.shape[-1] == 1 or im.shape[-1] == 3 + h, w = im.shape[:2] + h_new, w_new = size, size + if h > w: + h_new = size * h / w + else: + w_new = size * w / h + im = resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) + return im + + +def to_chw(im, order=(2, 0, 1)): + """ + Transpose the input image order. The image layout is HWC format + opened by cv2 or PIL. Transpose the input image to CHW layout + according the order (2,0,1). + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + im = to_chw(im) + + :param im: the input image with HWC layout. + :type im: ndarray + :param order: the transposed order. + :type order: tuple|list + """ + assert len(im.shape) == len(order) + im = im.transpose(order) + return im + + +def center_crop(im, size, is_color=True): + """ + Crop the center of image with size. + + Example usage: + + .. code-block:: python + im = center_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size. + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = (h - size) / 2 + w_start = (w - size) / 2 + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def random_crop(im, size, is_color=True): + """ + Randomly crop input image with size. + + Example usage: + + .. code-block:: python + im = random_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size. + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = np.random.randint(0, h - size + 1) + w_start = np.random.randint(0, w - size + 1) + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def left_right_flip(im): + """ + Flip an image along the horizontal direction. + Return the flipped image. + + Example usage: + + .. code-block:: python + im = left_right_flip(im) + + :paam im: input image with HWC layout + :type im: ndarray + """ + if len(im.shape) == 3: + return im[:, ::-1, :] + else: + return im[:, ::-1, :] + + +def simple_transform(im, resize_size, crop_size, is_train, is_color=True): + """ + Simply data argumentation for training. These operations include + resizing, croping and flipping. + + Example usage: + + .. code-block:: python + im = simple_transform(im, 256, 224, True) + + :param im: The input image with HWC layout. + :type im: ndarray + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = resize_short(im, resize_size) + if is_train: + im = random_crop(im, crop_size) + if np.random.randint(2) == 0: + im = left_right_flip(im) + else: + im = center_crop(im, crop_size) + im = to_chw(im) + + return im + + +def load_and_transform(filename, + resize_size, + crop_size, + is_train, + is_color=True): + """ + Load image from the input file `filename` and transform image for + data argumentation. Please refer to the `simple_transform` interface + for the transform operations. + + Example usage: + + .. code-block:: python + im = load_and_transform('cat.jpg', 256, 224, True) + + :param filename: The file name of input image. + :type filename: string + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = load_image(filename) + im = simple_transform(im, resize_size, crop_size, is_train, is_color) + return im diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py index b4bb38496937bb6fb520334331c619f9b6f64b51..139339902e9e2228f72068bed4c4ebe58ebc4428 100644 --- a/python/paddle/v2/inference.py +++ b/python/paddle/v2/inference.py @@ -88,7 +88,7 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): Infer a neural network by given neural network output and parameters. The user should pass either a batch of input data or reader method. - Example usages: + Example usage for sinlge output_layer: .. code-block:: python @@ -97,8 +97,19 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): input=SomeData) print result + Example usage for multiple outout_layers and fields: + + .. code-block:: python + + result = paddle.infer(output_layer=[prediction1, prediction2], + parameters=parameters, + input=SomeData, + field=[id, value]]) + print result + :param output_layer: output of the neural network that would be inferred - :type output_layer: paddle.v2.config_base.Layer + :type output_layer: paddle.v2.config_base.Layer or a list of + paddle.v2.config_base.Layer :param parameters: parameters of the neural network. :type parameters: paddle.v2.parameters.Parameters :param input: input data batch. Should be a python iterable object, and each @@ -112,7 +123,9 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): Note that `prob` only used when output_layer is beam_search or max_id. :type field: str - :return: a numpy array + :return: The prediction result. If there are multiple outout_layers and fields, + the return order is outout_layer1.field1, outout_layer2.field1, ..., + outout_layer1.field2, outout_layer2.field2 ... :rtype: numpy.ndarray """ diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index 5554a37df03cb31b37e9bbf68faa606d7c2cadde..eb02e53706b4834eb9dc75d0e3a809772b124725 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1 +1,2 @@ -add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py) +add_python_test(test_v2_api test_data_feeder.py test_parameters.py +test_layer.py test_rnn_layer.py test_topology.py test_image.py) diff --git a/python/paddle/v2/tests/cat.jpg b/python/paddle/v2/tests/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bc1fbbd371216b9904b522ed302700c79d2e4876 Binary files /dev/null and b/python/paddle/v2/tests/cat.jpg differ diff --git a/python/paddle/v2/tests/test_image.py b/python/paddle/v2/tests/test_image.py new file mode 100644 index 0000000000000000000000000000000000000000..b2d773510de28ca2614e95b465c73b82aa7b0463 --- /dev/null +++ b/python/paddle/v2/tests/test_image.py @@ -0,0 +1,42 @@ +# Copyright PaddlePaddle contributors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import numpy as np + +import paddle.v2.image as image + + +class Image(unittest.TestCase): + def test_resize_flip_chw(self): + # resize + im = image.load_image('cat.jpg') + im = image.resize_short(im, 256) + self.assertEqual(256, min(im.shape[:2])) + self.assertEqual(3, im.shape[2]) + + # flip + im = image.left_right_flip(im) + im2 = np.flip(im, 1) + self.assertEqual(im.all(), im2.all()) + + # to_chw + h, w, c = im.shape + im = image.to_chw(im) + self.assertEqual(c, im.shape[0]) + self.assertEqual(h, im.shape[1]) + self.assertEqual(w, im.shape[2]) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/setup.py.in b/python/setup.py.in index 5dfb46192ae54fdc36b0867312cf156aefb84f84..7d9438e3f8132c2a7fa4774750f5fd15f3beed14 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,6 +18,7 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", + "opencv-python", ], packages=packages, package_dir={