diff --git a/Dockerfile b/Dockerfile index 6d5797da9e06a514e03dee52bd248478fa41813f..ad0d086d3c65b5901178aa681aa36ccc0ea0c246 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # A image for building paddle binaries # Use cuda devel base image for both cpu and gpu environment -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 MAINTAINER PaddlePaddle Authors ARG UBUNTU_MIRROR @@ -23,11 +23,14 @@ ENV HOME /root COPY ./paddle/scripts/docker/root/ /root/ RUN apt-get update && \ - apt-get install -y git python-pip python-dev openssh-server bison && \ - apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ - apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ - apt-get install -y python-numpy python-matplotlib gcc g++ liblapack-dev liblapacke-dev && \ - apt-get install -y automake locales clang-format-3.8 swig doxygen && \ + apt-get install -y \ + git python-pip python-dev openssh-server bison \ + wget unzip tar xz-utils bzip2 gzip coreutils \ + curl sed grep graphviz libjpeg-dev zlib1g-dev \ + python-numpy python-matplotlib gcc g++ \ + automake locales clang-format-3.8 swig doxygen cmake \ + liblapack-dev liblapacke-dev libboost-dev \ + clang-3.8 llvm-3.8 libclang-3.8-dev && \ apt-get clean -y # git credential to skip password typing @@ -51,11 +54,12 @@ RUN pip install --upgrade pip && \ RUN apt-get install -y libssl-dev libffi-dev RUN pip install certifi urllib3[secure] -RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ - cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ - cd .. && rm -rf cmake-3.4.1 - -VOLUME ["/woboq_out"] +# Install woboq_codebrowser to /woboq +RUN git clone https://github.com/woboq/woboq_codebrowser /woboq && \ + (cd /woboq \ + cmake -DLLVM_CONFIG_EXECUTABLE=/usr/bin/llvm-config-3.8 \ + -DCMAKE_BUILD_TYPE=Release . \ + make) # Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service RUN mkdir /var/run/sshd diff --git a/RELEASE.cn.md b/RELEASE.cn.md new file mode 100755 index 0000000000000000000000000000000000000000..5deaf230a8f5dd3089993f0fc79b9460fd049750 --- /dev/null +++ b/RELEASE.cn.md @@ -0,0 +1,80 @@ +# v0.10.0版本 + +我们非常高兴发布了PaddlePaddle V0.10.0版，并开发了新的[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 + +- 旧的Python API由于难以学习和使用已经过时了。使用旧版本的API至少需要两份python文件，分别是定义数据生成器和定义网络拓扑结构的文件。用户通过运行`paddle_trainer`的C++程序来启动PaddlePaddle任务，该程序调用Python解释器来运行定义网络拓扑结构的文件，然后通过迭代加载数据生成器提供的小批量数据启动训练循环。这与Python的现代编辑方式不符，比如Jupyter Notebook。 + +- 新版的API被称为 *V2 API*，允许我们在单个.py文件中，通过编辑更短的Python程序来定义网络结构和数据。此外，该Python程序也可以在Jupyter Notebook中运行，因为PaddlePaddle可以作为共享库来被Python程序加载和使用。 + +基于新的API，我们提供了一个在线的学习文档 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 + +我们还致力于迭代更新新版API的在线文档，并将新版API引入分布式集群（包括MPI和Kubernetes）训练中。我们将在下一个版本中发布更多的内容。 + +## 新特点 + +* 发布新版[Python API](http://research.baidu.com/paddlepaddles-new-api-simplifies-deep-learning-programs/)。 +* 发布深度学习系列课程 [Deep Learning 101](http://book.paddlepaddle.org/index.en.html) 及其[中文版本](http://book.paddlepaddle.org/)。 +* 支持矩形输入的CNN。 +* 为seqlastin和seqfirstin提供stride pooling。 +* 在`trainer_config_helpers`中暴露`seq_concat_layer/seq_reshape_layer`。 +* 添加公共数据集包：CIFAR，MNIST，IMDB，WMT14，CONLL05，movielens，imikolov。 +* 针对Single Shot Multibox Detection增加 Prior box layer。 +* 增加光滑的L1损失。 +* 在V2 API中增加 data reader 创建器和修饰器。 +* 增加cmrnorm投影的CPU实现。 + + +## 改进 + +* 提供`paddle_trainer`的Python virtualenv支持。 +* 增加代码自动格式化的pre-commit hooks。 +* 升级protobuf到3.x版本。 +* 在Python数据生成器中提供一个检测数据类型的选项。 +* 加速GPU中average层的后向反馈计算。 +* 细化文档。 +* 使用Travis-CI检查文档中的死链接。 +* 增加解释`sparse_vector`的示例。 +* 在layer_math.py中添加ReLU。 +* 简化Quick Start示例中的数据处理流程。 +* 支持CUDNN Deconv。 +* 在v2 API中增加数据feeder。 +* 在情感分析示例的演示中增加对标准输入流中样本的预测。 +* 提供图像预处理的多进程接口。 +* 增加V1 API的基准文档。 +* 在`layer_math.py`中增加ReLU。 +* 提供公共数据集的自动下载包。 +* 将`Argument::sumCost`重新命名为`Argument::sum`，并暴露给python。 +* 为矩阵相关的表达式评估增加一个新的`TensorExpression`实现。 +* 增加延迟分配来优化批处理多表达式计算。 +* 增加抽象的类函数及其实现： + * `PadFunc` 和 `PadGradFunc`。 + * `ContextProjectionForwardFunc` 和 `ContextProjectionBackwardFunc`。 + * `CosSimBackward` 和 `CosSimBackwardFunc`。 + * `CrossMapNormalFunc` 和 `CrossMapNormalGradFunc`。 + * `MulFunc`。 +* 增加`AutoCompare`和`FunctionCompare`类，使得编写比较gpu和cpu版本函数的单元测试更容易。 +* 生成`libpaddle_test_main.a`并删除测试文件内的主函数。 +* 支持PyDataProvider2中numpy的稠密向量。 +* 清理代码库，删除一些复制粘贴的代码片段： + * 增加`SparseRowMatrix`的抽样类`RowBuffer`。 + * 清理`GradientMachine`的接口。 + * 在layer中增加`override`关键字。 + * 简化`Evaluator::create`，使用`ClassRegister`来创建`Evaluator`。 +* 下载演示的数据集时检查MD5校验。 +* 添加`paddle::Error`，用于替代Paddle中的`LOG(FATAL)`。 + + +## 错误修复 + +* 检查`recurrent_group`的layer输入类型。 +* 不要用.cu源文件运行`clang-format`。 +* 修复`LogActivation`的使用错误。 +* 修复运行`test_layerHelpers`多次的错误。 +* 修复seq2seq示例超出消息大小限制的错误。 +* 修复在GPU模式下dataprovider转换的错误。 +* 修复`GatedRecurrentLayer`中的错误。 +* 修复在测试多个模型时`BatchNorm`的错误。 +* 修复paramRelu在单元测试时崩溃的错误。 +* 修复`CpuSparseMatrix`编译时相关的警告。 +* 修复`MultiGradientMachine`在`trainer_count > batch_size`时的错误。 +* 修复`PyDataProvider2`阻止异步加载数据的错误。 diff --git a/doc/api/v1/data_provider/pydataprovider2_en.rst b/doc/api/v1/data_provider/pydataprovider2_en.rst index 30357be32538db4423ad0eaf899138256c84edc7..e8fb6292779790765154502bff319ea10ab1e70b 100644 --- a/doc/api/v1/data_provider/pydataprovider2_en.rst +++ b/doc/api/v1/data_provider/pydataprovider2_en.rst @@ -178,7 +178,7 @@ input_types +++++++++++ PaddlePaddle has four data types, and three sequence types. -The four data types are: +The four data types are: * :code:`dense_vector`: dense float vector. * :code:`sparse_binary_vector`: sparse binary vector, most of the value is 0, and @@ -231,7 +231,7 @@ Its parameters lists as follows: * :code:`is_train` is a bool parameter that indicates the DataProvider is used in training or testing. * :code:`file_list` is the list of all files. - + * User-defined parameters args can be set in training configuration. Note, PaddlePaddle reserves the right to add pre-defined parameter, so please diff --git a/doc/design/cluster_train/README.md b/doc/design/cluster_train/README.md index b88a8f382bfdbbf113d2bb99e12caaa09f9cd8c5..74961f80050c6b2723889b51416a2e8048174b00 100644 --- a/doc/design/cluster_train/README.md +++ b/doc/design/cluster_train/README.md @@ -15,7 +15,7 @@ This poses technical challenges to PaddlePaddle: A training job will be created once user asks Paddle cloud to train a model. The training job is made up of different processes that collaboratively consume data and produce a trained model. There are three kinds of processes: -1. the *master process*, which dispatches tasks to +1. the *master server process*, which dispatches tasks to 1. one or more *trainer processes*, which run distributed training and synchronize gradients/models via 1. one or more *parameter server processes*, where each holds a shard of the global model, and receive the uploaded gradients from every *trainer process*, so they can run the optimize functions to update their parameters. @@ -27,9 +27,9 @@ By coordinating these processes, PaddlePaddle supports use both Synchronize Stoc When training with sync SGD, parameter servers wait for all trainers to finish gradients update and then send the updated parameters to trainers, training can not proceed until the trainer received the updated parameters. This creates a synchronization point between trainers. When training with async SGD, each trainer upload gradient and download new parameters individually, without the synchronization with other trainers. Using asyc SGD will be faster in terms of time per pass, but have more noise in gradient since trainers are likely to have a stale model. -### Master Process +### Master Server Process -The master process will: +The master server process will: - Partition a dataset into [tasks](#task) and dispatch tasks to trainers. - Keep track of training progress on the dataset with [task queue](#task-queue). A training job will iterate on the dataset for a full pass until it goes into next pass. @@ -41,11 +41,11 @@ A task is a data shard to be trained. The total number of tasks will be much big #### Task Queue -The master process has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master process. Each master process has three task queues. +The master server has three task queues to track training progress. As illustrated in the graph below, Job A and Job B both have one master server. Each master server process has three task queues. -- The todo queue holds tasks to be dispatched. When a job starts, the master process fills in the todo queue with all tasks. +- The todo queue holds tasks to be dispatched. When a job starts, the master server fills in the todo queue with all tasks. - The pending queue holds tasks that are currently training by trainers. - the done queue holds tasks that are already trained. @@ -54,10 +54,10 @@ The life cycle of a single task is illustrated below: 1. When a new pass of training starts, all tasks will be placed in the todo queue. -1. The master process will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. -1. The trainer will work on its tasks and tell the master process once a task is completed. The master process will dispatch a new task to that trainer. -1. If a task timeout. the master process will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. -1. The master process will move completed task to the done queue. When the todo queue is empty, the master process will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. +1. The master server will dispatch few tasks to each trainer at a time, puts them in the pending queue and waits for completion. +1. The trainer will work on its tasks and tell the master server once a task is completed. The master server will dispatch a new task to that trainer. +1. If a task timeout. the master server will move it back to the todo queue. The timeout count will increase by one. If the timeout count is above a threshold, the task is likely to cause a trainer to crash, so it will be discarded. +1. The master server will move completed task to the done queue. When the todo queue is empty, the master server will start a new pass by moving all tasks in the done queue to todo queue and reset the timeout counter of all tasks to zero. ### Trainer Process @@ -93,7 +93,7 @@ The communication pattern between the trainers and the parameter servers depends ## Fault Tolerant -The training job will pause if the master processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). +The training job will pause if the master server processes is dead, or any of the parameter server process is dead. They will be started by [Kubernetes](https://kubernetes.io/) and recover in few minutes. Please refer to [fault recovery](#fault-recovery). The training job will continue to make progress if there is at least one training process running. The strategy depends on the type of optimization algorithm: @@ -113,7 +113,7 @@ Now we will introduce how each process recovers from a failure, the graph below -### Master Process +### Master Server Process When the master is started by the Kubernetes, it executes the following steps at startup: @@ -122,7 +122,7 @@ When the master is started by the Kubernetes, it executes the following steps at 1. Watches the trainer prefix keys `/trainer/` on etcd to find the live trainers. 1. Starts dispatching the tasks to the trainers, and updates task queue using an etcd transaction to ensure lock is held during the update. -When the master process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. +When the master server process is dead for any reason, Kubernetes will restart it. It will be online again with all states recovered from etcd in few minutes. ### Trainer Process @@ -132,7 +132,7 @@ When the trainer is started by the Kubernetes, it executes the following steps a 1. Generates a unique ID, and sets key `/trainer/` with its contact address as value. The key will be deleted when the lease expires, so the master will be aware of the trainer being online and offline. 1. Waits for tasks from the master to start training. -If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master process can discover the trainer again. +If trainer's etcd lease expires, it will try set key `/trainer/` again so that the master server can discover the trainer again. When a trainer fails, Kuberentes would try to restart it. The recovered trainer would fetch tasks from the TODO queue and go on training. diff --git a/doc/design/cluster_train/data_dispatch.md b/doc/design/cluster_train/data_dispatch.md index a84fcd851d92b58e0596e60f586b139eab98a177..1f5d22ff5e6abcb576d16cbe7391da1967a1ab8e 100644 --- a/doc/design/cluster_train/data_dispatch.md +++ b/doc/design/cluster_train/data_dispatch.md @@ -27,10 +27,11 @@ ### 文件预处理 + 在开始训练之前, 数据集需要预先被转换成PaddlePaddle分布式训练使用的存储格[RecordIO](https://github.com/PaddlePaddle/Paddle/issues/1947)。我们提供两个转换方式： -- 提供给用户本地转换的库，用户可以编写程序完成转换。 -- 用户可以上传自己的数据集，在集群运行MapReduce job完成转换。 +1. 用户在本地转换好再上传 +1. 用户上传数据后，在机群上运行转换程序 转换生成的文件名会是以下格式： @@ -98,6 +99,7 @@ random_images-00099-of-00099 #### 进行训练 + PaddlePaddle提供专用的[data reader creator](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md#python-data-reader-design-doc)，生成给定`RecordIO`文件对应的data reader。**无论在本地还是在云端，reader的使用方式都是一致的**： ```python diff --git a/doc/design/cluster_train/master_server.md b/doc/design/cluster_train/master_server.md new file mode 100644 index 0000000000000000000000000000000000000000..bb8307652587b4dc56cd668a3a5e64722734d194 --- /dev/null +++ b/doc/design/cluster_train/master_server.md @@ -0,0 +1,91 @@ +# Design Doc: Master Server + +For an overview of master server's role, please refer to [distributed training design doc](./README.md). In this design doc we will discuss the master server in more details. The master will be implemented in [Go](https://golang.org/). + +## Dataset + + + +A dataset is a list of files in *RecordIO* format. A RecordIO file consists of chunks, whereas each chunk consists some records. + +## Task Queue + +As mentioned in [distributed training design doc](./README.md), a *task* is a data shard that the master server assigns to the trainer process to train on. A task consists of one or multiple *blocks* from one or multiple files. The master server maintains *task queues* to track the training progress. + +### Task Queue Creation + +1. Each trainer will make an RPC call (using Go's [rpc](https://golang.org/pkg/net/rpc/) package) to the master server, telling it the RecordIO files representing the dataset specified by the user. Since every trainer will tell the master server the same dataset, only the first RPC call will be honored. + + The RPC interface is: + ```go + func (m *RPCServer) ReportDataset(Paths []string, dummy *int) error { + } + ``` +1. The master server will scan through each RecordIO file to generate the *block index* and know how many blocks does each file have. A block can be referenced by the file path and the index of the block within the file. The block index is in memory data structure that enables fast access to each block, and the index of the block with the file is an integer start from 0, representing the n-th block within the file. + + The definition of the block is: + ```go + type Block struct { + Idx int // index of the block within the file + Path string + Index recordio.Index // block index + } + ``` +1. Blocks are grouped into tasks, and tasks are filled into the todo queue. The pending queue and the done queue are initialized with no element. + + The definition of the task is: + ```go + type Task struct { + Index int + Blocks []Block + } + ``` + + The elements in the tasks queues is of type `TaskEntry`, containing a timeout counter (described in [task retry logic](#task-retry-logic)), and a task: + ```go + type TaskEntry struct { + NumTimeout int + Task Task + } + ``` + + The definition of task queues is: + ```go + type TaskQueues struct { + Todo []TaskEntry + Pending map[int]TaskEntry // map from task index to task entry + Done []TaskEntry + } + ``` + +### Task Queue Persistence + +The task queues need to be persisted on [etcd](https://github.com/coreos/etcd) for fault recovery. Since the task queues only change once a task is completed or timed out, which is not very frequent, we can afford to synchronize with etcd every time the task queues change. + +We will serialize the task queues data structure with [gob encoding](https://golang.org/pkg/encoding/gob/), compress with gzip, and save into etcd synchronously under key `/task_queues`. + +### Task Dispatch + +The trainer will make an RPC call to master to get a new task when: + +- the trainer first started, or +- the trainer finishes a task. + +The RPC interface is: +```go +func (m *RPCServer) GetTask(finished *Task, result *Task) error { +} +``` +Argument `finished` will be `nil` when the trainer is just started. + +During the RPC call the master will do the following: + +- Make a copy of the task queues, and update the copy reflecting the finished tasks and the new pending tasks. +- Synchronize the copy of task queues with etcd using a transaction conditioned on holding the master lock. +- Replace the task queues with the copy and report to the trainer with the new tasks if succeeded, or discard the copy and report the error to the trainer if failed. + +### Task Retry Logic + +When a task is dispatched to the trainer, the master will schedule a function for execution after the timeout duration (based on the moving average of task completion time). If the task entry in still in the pending queue, its timeout counter will increase by one, and the task will be moved to todo queue. If the timeout counter is above the threshold, the master will log the error and discard the task. + +Please note that since a timed out task could be completed after it has been dispatched for retry, so it is possible for a task to be processed multiple times. We do not try to prevent it from happening since it's fine to train on the same task multiple times due to the stochastic nature of the stochastic gradient decent algorithm. diff --git a/doc/design/cluster_train/src/dataset.graffle b/doc/design/cluster_train/src/dataset.graffle new file mode 100644 index 0000000000000000000000000000000000000000..c10a423ed16a23229a9ee33d11bfc82bb59646c8 Binary files /dev/null and b/doc/design/cluster_train/src/dataset.graffle differ diff --git a/doc/design/cluster_train/src/dataset.png b/doc/design/cluster_train/src/dataset.png new file mode 100644 index 0000000000000000000000000000000000000000..2fb7f1cce3b6dd21489392557826e95a9f207c34 Binary files /dev/null and b/doc/design/cluster_train/src/dataset.png differ diff --git a/doc/getstarted/concepts/src/train.py b/doc/getstarted/concepts/src/train.py new file mode 100644 index 0000000000000000000000000000000000000000..679d0a931a7d650108ea89a04080a55d2976f72e --- /dev/null +++ b/doc/getstarted/concepts/src/train.py @@ -0,0 +1,52 @@ +import paddle.v2 as paddle +import numpy as np + +# init paddle +paddle.init(use_gpu=False) + +# network config +x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) +y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) +y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) +cost = paddle.layer.mse_cost(input=y_predict, label=y) + +# create parameters +parameters = paddle.parameters.create(cost) +# create optimizer +optimizer = paddle.optimizer.Momentum(momentum=0) +# create trainer +trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + + +# event_handler to print training info +def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % (event.pass_id, event.batch_id, + event.cost) + + +# define training dataset reader +def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + + return reader + + +# define feeding map +feeding = {'x': 0, 'y': 1} + +# training +trainer.train( + reader=paddle.batch( + train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) diff --git a/doc/getstarted/concepts/use_concepts_cn.rst b/doc/getstarted/concepts/use_concepts_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..e63ca11102c8ce457afcc3c262fa5f159361c01d --- /dev/null +++ b/doc/getstarted/concepts/use_concepts_cn.rst @@ -0,0 +1,150 @@ +############ +基本使用概念 +############ + +PaddlePaddle是源于百度的一个深度学习平台。PaddlePaddle为深度学习研究人员提供了丰富的API，可以轻松地完成神经网络配置，模型训练等任务。 +这里将介绍PaddlePaddle的基本使用概念，并且展示了如何利用PaddlePaddle来解决一个经典的线性回归问题。 +在使用该文档之前，请参考 `安装文档 <../build_and_install/index_cn.html>`_ 完成PaddlePaddle的安装。 + + +配置网络 +============ + +加载PaddlePaddle +---------------------- + +在进行网络配置之前，首先需要加载相应的Python库，并进行初始化操作。 + +.. code-block:: bash + + import paddle.v2 as paddle + import numpy as np + paddle.init(use_gpu=False) + + +搭建神经网络 +----------------------- + +搭建神经网络就像使用积木搭建宝塔一样。在PaddlePaddle中，layer是我们的积木，而神经网络是我们要搭建的宝塔。我们使用不同的layer进行组合，来搭建神经网络。 +宝塔的底端需要坚实的基座来支撑，同样，神经网络也需要一些特定的layer作为输入接口，来完成网络的训练。 + +例如，我们可以定义如下layer来描述神经网络的输入： + +.. code-block:: bash + + x = paddle.layer.data(name='x', type=paddle.data_type.dense_vector(2)) + y = paddle.layer.data(name='y', type=paddle.data_type.dense_vector(1)) + +其中x表示输入数据是一个维度为2的稠密向量，y表示输入数据是一个维度为1的稠密向量。 + +PaddlePaddle支持不同类型的输入数据，主要包括四种类型，和三种序列模式。 + +四种数据类型： + +* dense_vector：稠密的浮点数向量。 +* sparse_binary_vector：稀疏的01向量，即大部分值为0，但有值的地方必须为1。 +* sparse_float_vector：稀疏的向量，即大部分值为0，但有值的部分可以是任何浮点数。 +* integer：整数标签。 + +三种序列模式： + +* SequenceType.NO_SEQUENCE：不是一条序列 +* SequenceType.SEQUENCE：是一条时间序列 +* SequenceType.SUB_SEQUENCE： 是一条时间序列，且序列的每一个元素还是一个时间序列。 + +不同的数据类型和序列模式返回的格式不同，列表如下： + ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| | NO_SEQUENCE | SEQUENCE | SUB_SEQUENCE | ++======================+=====================+===================================+================================================+ +| dense_vector | [f, f, ...] | [[f, ...], [f, ...], ...] | [[[f, ...], ...], [[f, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_binary_vector | [i, i, ...] | [[i, ...], [i, ...], ...] | [[[i, ...], ...], [[i, ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| sparse_float_vector | [(i,f), (i,f), ...] | [[(i,f), ...], [(i,f), ...], ...] | [[[(i,f), ...], ...], [[(i,f), ...], ...],...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ +| integer_value | i | [i, i, ...] | [[i, ...], [i, ...], ...] | ++----------------------+---------------------+-----------------------------------+------------------------------------------------+ + +其中，f代表一个浮点数，i代表一个整数。 + +注意：对sparse_binary_vector和sparse_float_vector，PaddlePaddle存的是有值位置的索引。例如， + +- 对一个5维非序列的稀疏01向量 ``[0, 1, 1, 0, 0]`` ，类型是sparse_binary_vector，返回的是 ``[1, 2]`` 。 +- 对一个5维非序列的稀疏浮点向量 ``[0, 0.5, 0.7, 0, 0]`` ，类型是sparse_float_vector，返回的是 ``[(1, 0.5), (2, 0.7)]`` 。 + + +在定义输入layer之后，我们可以使用其他layer进行组合。在组合时，需要指定layer的输入来源。 + +例如，我们可以定义如下的layer组合： + +.. code-block:: bash + + y_predict = paddle.layer.fc(input=x, size=1, act=paddle.activation.Linear()) + cost = paddle.layer.mse_cost(input=y_predict, label=y) + +其中，x与y为之前描述的输入层；而y_predict是接收x作为输入，接上一个全连接层；cost接收y_predict与y作为输入，接上均方误差层。 + +最后一层cost中记录了神经网络的所有拓扑结构，通过组合不同的layer，我们即可完成神经网络的搭建。 + + +训练模型 +============ + +在完成神经网络的搭建之后，我们首先需要根据神经网络结构来创建所需要优化的parameters，并创建optimizer。 +之后，我们可以创建trainer来对网络进行训练。 + +.. code-block:: bash + + parameters = paddle.parameters.create(cost) + optimizer = paddle.optimizer.Momentum(momentum=0) + trainer = paddle.trainer.SGD(cost=cost, + parameters=parameters, + update_equation=optimizer) + +其中，trainer接收三个参数，包括神经网络拓扑结构、神经网络参数以及迭代方程。 + +在搭建神经网络的过程中，我们仅仅对神经网络的输入进行了描述。而trainer需要读取训练数据进行训练，PaddlePaddle中通过reader来加载数据。 + +.. code-block:: bash + + # define training dataset reader + def train_reader(): + train_x = np.array([[1, 1], [1, 2], [3, 4], [5, 2]]) + train_y = np.array([-2, -3, -7, -7]) + def reader(): + for i in xrange(train_y.shape[0]): + yield train_x[i], train_y[i] + return reader + +最终我们可以调用trainer的train方法启动训练： + +.. code-block:: bash + + # define feeding map + feeding = {'x': 0, 'y': 1} + + # event_handler to print training info + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 1 == 0: + print "Pass %d, Batch %d, Cost %f" % ( + event.pass_id, event.batch_id, event.cost) + # training + trainer.train( + reader=paddle.batch(train_reader(), batch_size=1), + feeding=feeding, + event_handler=event_handler, + num_passes=100) + +关于PaddlePaddle的更多使用方法请参考 `进阶指南 <../../howto/index_cn.html>`_。 + +线性回归完整示例 +============== + +下面给出在三维空间中使用线性回归拟合一条直线的例子： + +.. literalinclude:: src/train.py + :linenos: + +有关线性回归的实际应用，可以参考PaddlePaddle book的 `第一章节 `_。 \ No newline at end of file diff --git a/doc/getstarted/index_cn.rst b/doc/getstarted/index_cn.rst index cadf092f8f42ca16bbeb23bd21e0d018af8e43cc..0cb27f802c40ef123fdc9c6799aad3b2a5f554c0 100644 --- a/doc/getstarted/index_cn.rst +++ b/doc/getstarted/index_cn.rst @@ -5,5 +5,6 @@ :maxdepth: 1 build_and_install/index_cn.rst + concepts/use_concepts_cn.rst - `深度学习入门课程 `_ diff --git a/doc/howto/index_cn.rst b/doc/howto/index_cn.rst index 5b84eea491f874459ed2071e4c942657cdc9b18b..26449a6365843b526b3ac3111b337d2f17524c9d 100644 --- a/doc/howto/index_cn.rst +++ b/doc/howto/index_cn.rst @@ -8,7 +8,6 @@ :maxdepth: 1 usage/cmd_parameter/index_cn.rst - usage/concepts/use_concepts_cn.rst usage/cluster/cluster_train_cn.md usage/k8s/k8s_basis_cn.md usage/k8s/k8s_cn.md diff --git a/doc/howto/usage/concepts/src/pserver_topology.dot b/doc/howto/usage/concepts/src/pserver_topology.dot deleted file mode 100644 index 9ff658b8495030f322d4f553f3bf72ddf8d3a578..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/src/pserver_topology.dot +++ /dev/null @@ -1,68 +0,0 @@ -graph pp_topology { - rankdir=BT; - subgraph cluster_node0 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器0" - - pserver0 [label="Parameter \n Server 0"] - trainer0 [label="Trainer 0"] - } - subgraph cluster_node1 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器1" - - pserver1 [label="Parameter \n Server 1"] - trainer1 [label="Trainer 1"] - } - - subgraph cluster_node2 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器2" - - pserver2 [label="Parameter \n Server 2"] - trainer2 [label="Trainer 2"] - } - - subgraph cluster_node3 { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "机器3" - - pserver3 [label="Parameter \n Server 3"] - trainer3 [label="Trainer 3"] - } - - data [label="数据", shape=hexagon] - - trainer0 -- pserver0 - trainer0 -- pserver1 - trainer0 -- pserver2 - trainer0 -- pserver3 - - trainer1 -- pserver0 - trainer1 -- pserver1 - trainer1 -- pserver2 - trainer1 -- pserver3 - - trainer2 -- pserver0 - trainer2 -- pserver1 - trainer2 -- pserver2 - trainer2 -- pserver3 - - trainer3 -- pserver0 - trainer3 -- pserver1 - trainer3 -- pserver2 - trainer3 -- pserver3 - - data -- trainer0 - data -- trainer1 - data -- trainer2 - data -- trainer3 -} diff --git a/doc/howto/usage/concepts/src/trainer_config.py b/doc/howto/usage/concepts/src/trainer_config.py deleted file mode 100644 index 3eccbd7bc11f4865130286de718d1be74e4d1722..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/src/trainer_config.py +++ /dev/null @@ -1,29 +0,0 @@ -from paddle.trainer_config_helpers import * - -define_py_data_sources2( - train_list='train.list', - test_list='test.list', - module='provider', - obj='process') -settings( - batch_size=128, - learning_rate=1e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(0.5)) - -img = data_layer(name='pixel', size=28 * 28) - -hidden1 = simple_img_conv_pool( - input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) - -hidden2 = fc_layer( - input=hidden1, - size=200, - act=TanhActivation(), - layer_attr=ExtraAttr(drop_rate=0.5)) -predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) - -outputs( - classification_cost( - input=predict, label=data_layer( - name='label', size=10))) diff --git a/doc/howto/usage/concepts/use_concepts_cn.rst b/doc/howto/usage/concepts/use_concepts_cn.rst deleted file mode 100644 index fa334bcbb9e29d6943def7c35fa53e1b9262d29c..0000000000000000000000000000000000000000 --- a/doc/howto/usage/concepts/use_concepts_cn.rst +++ /dev/null @@ -1,139 +0,0 @@ -############ -基本使用概念 -############ - -PaddlePaddle是一个深度学习框架，支持单机模式和多机模式。 - -单机模式用命令 ``paddle train`` 可以启动一个trainer进程，单机训练通常只包括一个trainer进程。如果数据规模比较大，希望加速训练，可以启动分布式作业。一个分布式作业里包括若干trainer进程和若干Parameter Server（或称pserver）进程。用命令 ``paddle pserver`` 可以启动 pserver 进程，pserver进程用于协调多个trainer进程之间的通信。 - -本文首先介绍trainer进程中的一些使用概念，然后介绍pserver进程中概念。 - -.. contents:: - -系统框图 -======== - -下图描述了用户使用框图，PaddlePaddle的trainer进程里内嵌了Python解释器，trainer进程可以利用这个解释器执行Python脚本，Python脚本里定义了模型配置、训练算法、以及数据读取函数。其中，数据读取程序往往定义在一个单独Python脚本文件里，被称为数据提供器（DataProvider），通常是一个Python函数。模型配置、训练算法通常定义在另一单独Python文件中, 称为训练配置文件。下面将分别介绍这两部分。 - -.. graphviz:: - - digraph pp_process { - rankdir=LR; - config_file [label="用户神经网络配置"]; - subgraph cluster_pp { - style=filled; - color=lightgrey; - node [style=filled, color=white, shape=box]; - label = "PaddlePaddle C++"; - py [label="Python解释器"]; - } - data_provider [label="用户数据解析"]; - config_file -> py; - py -> data_provider [dir="back"]; - } - -数据提供器 -========== - -DataProvider是PaddlePaddle系统的数据提供器，将用户的原始数据转换成系统可以识别的数据类型。每当系统需要新的数据训练时, trainer进程会调用DataProvider函数返回数据。当所有数据读取完一轮后，DataProvider返回空数据，通知系统一轮数据读取结束，并且系统每一轮训练开始时会重置DataProvider。需要注意的是，DataProvider是被系统调用，而不是新数据驱动系统，一些随机化噪声添加都应该在DataProvider中完成。 - -在不同的应用里，训练数据的格式往往各不相同。因此，为了用户能够灵活的处理数据，我们提供了Python处理数据的接口，称为 ``PyDataProvider`` 。在 ``PyDataProvider`` 中，系统C++模块接管了shuffle、处理batch、GPU和CPU通信、双缓冲、异步读取等问题，一些情况下(如：``min_pool_size=0``)需要Python接口里处理shuffle，可以参考 :ref:`api_pydataprovider2` 继续深入了解。 - - -训练配置文件 -============ - -训练配置文件主要包括数据源、优化算法、网络结构配置三部分。 其中数据源配置与DataProvider的关系是：DataProvider里定义数据读取函数，训练配置文件的数据源配置中指定DataProvider文件名字、生成数据函数接口，请不要混淆。 - -一个简单的训练配置文件为： - -.. literalinclude:: src/trainer_config.py - :linenos: - -文件开头 ``from paddle.trainer_config_helpers import *`` ，是因为PaddlePaddle配置文件与C++模块通信的最基础协议是protobuf，为了避免用户直接写复杂的protobuf string，我们为用户定以Python接口来配置网络，该Python代码可以生成protobuf包，这就是 :ref:`api_trainer_config` 的作用。因此，在文件的开始，需要import这些函数。 这个包里面包含了模型配置需要的各个模块。 - -下面分别介绍数据源配置、优化算法配置、网络结构配置这三部分该概念。 - -数据源配置 ----------- - -使用 ``PyDataProvider2`` 的函数 ``define_py_data_sources2`` 配置数据源。``define_py_data_sources2`` 里通过train_list和test_list指定是训练文件列表和测试文件列表。 如果传入字符串的话，是指一个数据列表文件。这个数据列表文件中包含的是每一个训练或者测试文件的路径。如果传入一个list的话，则会默认生成一个list文件，再传入给train.list或者test.list。 - -``module`` 和 ``obj`` 指定了DataProvider的文件名和返回数据的函数名。更详细的使用，请参考 :ref:`api_pydataprovider2` 。 - -优化算法配置 ------------- - -通过 :ref:`api_trainer_config_helpers_optimizers_settings` 接口设置神经网络所使用的训练参数和 :ref:`api_trainer_config_helpers_optimizers` ，包括学习率、batch_size、优化算法、正则方法等，具体的使用方法请参考 :ref:`api_trainer_config_helpers_optimizers_settings` 文档。 - -网络结构配置 ------------- - -神经网络配置主要包括网络连接、激活函数、损失函数、评估器。 - -- 网络连接： 主要由Layer组成，每个Layer返回的都是一个 ``LayerOutput`` 对象，Layer里面可以定义参数属性、激活类型等。 - - 为了更灵活的配置，PaddlePaddle提供了基于 Projection 或者 Operator 的配置，这两个需要与 ``mixed_layer`` 配合使用。这里简单介绍Layer、Projection、Operator的概念: - - - Layer: 神经网络的某一层，可以有可学习的参数，一般是封装了许多复杂操作的集合。 - - Projection：需要与 ``mixed_layer`` 配合使用，含可学习参数。 - - Operator： 需要与 ``mixed_layer`` 配合使用，不含可学习参数，输入全是其他Layer的输出。 - - - 这个配置文件网络由 ``data_layer`` 、 ``simple_img_conv_pool`` 、 ``fc_layer`` 组成。 - - - :ref:`api_trainer_config_helpers_layers_data_layer` ： 通常每个配置文件都会包括 ``data_layer`` ，定义输入数据大小。 - - :ref:`api_trainer_config_helpers_network_simple_img_conv_pool` ：是一个组合层，包括了图像的卷积 (convolution)和池化(pooling)。 - - :ref:`api_trainer_config_helpers_layers_fc_layer` ：全连接层，激活函数为Softmax，这里也可叫分类层。 - -- 损失函数和评估器：损失函数即为网络的优化目标，评估器可以评价模型结果。 - - PaddlePaddle包括很多损失函数和评估起，详细可以参考 :ref:`api_trainer_config_helpers_layers_cost_layers` 和 :ref:`api_trainer_config_helpers_evaluators` 。这里 ``classification_cost`` 默认使用多类交叉熵损失函数和分类错误率统计评估器。 - -- ``outputs``: 标记网络输出的函数为 ``outputs`` 。 - - 训练阶段，网络的输出为神经网络的优化目标；预测阶段，网络的输出也可通过 ``outputs`` 标记。 - - -这里对 ``mixed_layer`` 稍做详细说明， 该Layer将多个输入(Projection 或 Operator)累加求和，具体计算是通过内部的 Projection 和 Operator 完成，然后加 Bias 和 activation 操作， - -例如，和 ``fc_layer`` 同样功能的 ``mixed_layer`` 是: - -.. code-block:: python - - data = data_layer(name='data', size=200) - with mixed_layer(size=200) as out: - out += full_matrix_projection(input=data) - -PaddlePaddle 可以使用 ``mixed layer`` 配置出非常复杂的网络，甚至可以直接配置一个完整的LSTM。用户可以参考 :ref:`api_trainer_config_helpers_layers_mixed_layer` 的相关文档进行配置。 - - -分布式训练 -========== - -PaddlePaddle多机采用经典的 Parameter Server 架构对多个节点的 trainer 进行同步。多机训练的经典拓扑结构如下\: - -.. graphviz:: src/pserver_topology.dot - -图中每个灰色方块是一台机器，在每个机器中，先使用命令 ``paddle pserver`` 启动一个pserver进程，并指定端口号，可能的参数是\: - -.. code-block:: bash - - paddle pserver --port=5000 --num_gradient_servers=4 --tcp_rdma='tcp' --nics='eth0' - -* ``--port=5000`` : 指定 pserver 进程端口是 5000 。 -* ``--gradient_servers=4`` : 有四个训练进程(PaddlePaddle 将 trainer 也称作 GradientServer ，因为其为负责提供Gradient) 。 -* ``--tcp_rdma='tcp' --nics=`eth0```: 指定以太网类型为TCP网络，指定网络接口名字为eth0。 - -启动之后 pserver 进程之后，需要启动 trainer 训练进程，在各个机器上运行如下命令\: - -.. code-block:: bash - - paddle train --port=5000 --pservers=192.168.100.101,192.168.100.102,192.168.100.103,192.168.100.104 --config=... - -对于简单的多机协同训练使用上述方式即可。另外，pserver/train 通常在高级情况下，还需要设置下面两个参数\： - -* --ports_num\: 一个 pserver 进程共绑定多少个端口用来做稠密更新，默认是1。 -* --ports_num_for_sparse\: 一个pserver进程共绑定多少端口用来做稀疏更新，默认是0。 - -使用手工指定端口数量，是因为Paddle的网络通信中，使用了 int32 作为消息长度，比较容易在大模型下溢出。所以，在 pserver 进程中可以启动多个子线程去接受 trainer 的数据，这样单个子线程的长度就不会溢出了。但是这个值不可以调的过大，因为增加这个值，对性能尤其是内存占用有一定的开销，另外稀疏更新的端口如果太大的话，很容易导致某一个参数服务器没有分配到任何参数。 diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index eff296bcb0174c71e05ee169c85236343a3e1164..c6fd9cc54ae3a671c5bdcf54cbaa873c59280694 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -9,6 +9,14 @@ add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) +find_package(boost QUIET) + +if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + add_subdirectory(majel) +endif() + if(WITH_C_API) add_subdirectory(capi) endif() diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 4d0dacae9058f94e584f313c9d0e31b5af09e82d..1cec77c0cae6ffbf7a1ca22092e8e41a6f9f0fc5 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -92,7 +92,9 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so # TODO(yuyang18) : make wheel name calculated by cmake add_custom_target(python_api_wheel ALL DEPENDS ${PROJ_ROOT}/paddle/py_paddle/_swig_paddle.so) -install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ DESTINATION opt/paddle/share/wheels) +install(DIRECTORY ${CMAKE_SOURCE_DIR}/paddle/dist/ + DESTINATION opt/paddle/share/wheels +) if(WITH_TESTING) IF(NOT PY_PIP_FOUND) diff --git a/paddle/majel/.gitignore b/paddle/majel/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1f5acdebb56971202b63d2485e2ac5042786f13c --- /dev/null +++ b/paddle/majel/.gitignore @@ -0,0 +1,2 @@ +build +third-party \ No newline at end of file diff --git a/paddle/majel/CMakeLists.txt b/paddle/majel/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c91fa72da4e5f7b19d0f3141799729e8a6375d3 --- /dev/null +++ b/paddle/majel/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.0) + +if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})") +else() + # find #include + get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) + include_directories(${PARENT_DIR}) + + # find cmake directory modules + get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + + # enable c++11 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + + # enable gtest + set(THIRD_PARTY_PATH ${CMAKE_CURRENT_SOURCE_DIR}/third_party) + set(WITH_TESTING ON) + include(external/gtest) +endif() + +########################### Build Majel ############################# +set(MAJEL_CXX_FILES place.cpp) +set(MAJEL_CUDA_FILES "") + +if(CUDA_FOUND) + cuda_add_library(majel ${MAJEL_CUDA_FILES} ${MAJEL_CXX_FILES}) +else() + add_library(majel ${MAJEL_CXX_FILES}) +endif() +##################################################################### + +add_subdirectory(test) diff --git a/paddle/majel/README.md b/paddle/majel/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5539853056797284ca1fa5ef5ab16fa0059907f0 --- /dev/null +++ b/paddle/majel/README.md @@ -0,0 +1,126 @@ +# Tensor: An Unified Data Type in PaddlePaddle + +## Pain Point + +In this week, we discussed several potential weaknesses of PaddlePaddle caused by rapid iteration and development to promote new business products on the line in recent four years. For instance, current Matrix/Vector implementation in PaddlePaddle are long and tedious to read, which interfered seriously with the contribution of both fresh and professional engineers. More seriously for this issue, it will also become too challenging to maintain over time. + + +## Learn from Majel + +Consequently, we decide to refactor PaddlePaddle step-by-step. First, refactor and replace Matrix/Vector to Tensor, a modern terminology in the deep learning system. Fortunately, we can learn from Majel how to define a Tensor. + +To simplify heterogeneous resource allocation in any dimensions (1-9) and types (double, float, float16), Majel consists of several primitives such as `Dim`, `Place` and `Array`, all of them are standard C++ class templates. + +1. `Place`: memory location [i.e. CPU/GPU]. +2. `Allocation`: heterogeneous resource allocator [i.e. 20MB in GPU]. +3. `Dim`: size of each dimension. [i.e. Dim<4>({10, 2, 5, 1})] +4. `Array`: dynamic array consists of `Place`, `Dim`, and a pointer to memory. + +If you dig deeper into Majel source code, you will find Majel heavily use `boost.variant`. The variant class template is a safe, generic, stack-based discriminated union container, **offering a simple solution for manipulating an object from a heterogeneous set of types in a uniform manner**. Whereas standard containers such as std::vector may be thought of as "multi-value, single type," variant is "multi-type, single value." + +As a simple example, consider the following: + +```c++ +#include "boost/variant.hpp" +#include + +class my_visitor : public boost::static_visitor +{ +public: + int operator()(int i) const + { + return i; + } + + int operator()(const std::string & str) const + { + return str.length(); + } +}; + +int main() +{ + boost::variant< int, std::string > u("hello world"); + std::cout << u; // output: hello world + + int result = boost::apply_visitor( my_visitor(), u ); + std::cout << result; // output: 11 (i.e., length of "hello world") +} +``` + +In Majel, `DDimVar` is derived from `Dim`, `DArrayVar` is from `Array`. + +```c++ +template +struct Dim { +... +int head; +Dim tail; +} +``` + +```c++ +template +class Array : public Buffer { + ... +private: + Dim size_; + Dim stride_; + T* ptr_; +}; +``` + +```c++ +typedef boost::variant Place; +typedef boost::variant, Dim<2>, Dim<3>, Dim<4>, Dim<5>, + Dim<6>, Dim<7>, Dim<8>, Dim<9>> DDimVar; +typedef boost::variant< + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array, + + Array, + Array, + Array, + Array > DArrayVar; +``` + +Because `variant` may be thought of as "multi-type, single value", we can utilize it to implement unified interfaces for PaddlePaddle. + +## implement Tensor in Paddle + +Before writing code, please make sure you already look through Majel Source Code and grabbed the design philosophy of `DArray` in Majel. + +To assign subtasks to our colleagues, we have to discuss how to divide it to independent subtasks. + +- [ ] 1. First, we need to consider the third-party dependencies in Majel. + + Majel heavily use `boost.variant`, but we don't want to integrate `boost` into PaddlePaddle. It's better to replace boost using the lightweight implementation. https://github.com/mapbox/variant Mapbox variant has the same speedy performance of `boost::variant `but is faster to compile, results in smaller binaries, and has no dependencies. + +> @gangliao + +- [ ] 2. Re-implement `Place` and `Allocation/Memory` + + I found @wangkuiyi submitted a pull request includes `Place`. @gangliao and @qijun could re-implement `Allocation`, because we have the GPU development experience before joining Paddle team. + +> @wangkuiyi @gangliao @qijun + +- [ ] 3. Re-implement `Dim`. + + `Dim` is an excellent implementation in Majel. + +> ??? + +- [ ] 4. Re-implement `Array/Tensor`. + +> Prerequisites: 1 - 3 + +- [ ] 5. Re-implement fundamental operators for `Array/Tensor`. + +> Prerequisites: 1 - 4 diff --git a/paddle/majel/place.cpp b/paddle/majel/place.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eecd8e5b730704258d2bd7d98a75a0a80e13a797 --- /dev/null +++ b/paddle/majel/place.cpp @@ -0,0 +1,49 @@ +#include + +namespace majel { + +namespace detail { + +class PlacePrinter : public boost::static_visitor<> { +private: + std::ostream& os_; + +public: + PlacePrinter(std::ostream& os) : os_(os) {} + + void operator()(const CpuPlace&) { os_ << "CpuPlace"; } + + void operator()(const GpuPlace& p) { os_ << "GpuPlace(" << p.device << ")"; } +}; + +} // namespace majel + +static Place the_default_place; + +void set_place(const Place& place) { the_default_place = place; } + +const Place& get_place() { return the_default_place; } + +const GpuPlace default_gpu() { return GpuPlace(0); } + +const CpuPlace default_cpu() { return CpuPlace(); } + +bool is_gpu_place(const Place& p) { + return boost::apply_visitor(IsGpuPlace(), p); +} + +bool is_cpu_place(const Place& p) { + return !boost::apply_visitor(IsGpuPlace(), p); +} + +bool places_are_same_class(const Place& p1, const Place& p2) { + return is_gpu_place(p1) == is_gpu_place(p2); +} + +std::ostream& operator<<(std::ostream& os, const majel::Place& p) { + majel::detail::PlacePrinter printer(os); + boost::apply_visitor(printer, p); + return os; +} + +} // namespace majel diff --git a/paddle/majel/place.h b/paddle/majel/place.h new file mode 100644 index 0000000000000000000000000000000000000000..ad3dc3fe0b80ac5dc10a59910c580d7912469cd4 --- /dev/null +++ b/paddle/majel/place.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include + +namespace majel { + +struct CpuPlace { + CpuPlace() {} // WORKAROUND: for some reason, omitting this constructor + // causes errors with boost 1.59 and OSX + // needed for variant equality comparison + inline bool operator==(const CpuPlace&) const { return true; } + + inline bool operator!=(const CpuPlace&) const { return false; } +}; + +struct GpuPlace { + GpuPlace(int d) : device(d) {} + + // needed for variant equality comparison + inline bool operator==(const GpuPlace& o) const { return device == o.device; } + + inline bool operator!=(const GpuPlace& o) const { return !(*this == o); } + + GpuPlace() : GpuPlace(0) {} + int device; +}; + +class IsGpuPlace : public boost::static_visitor { +public: + bool operator()(const CpuPlace&) const { return false; } + + bool operator()(const GpuPlace& gpu) const { return true; } +}; + +typedef boost::variant Place; + +void set_place(const Place&); + +const Place& get_place(); + +const GpuPlace default_gpu(); +const CpuPlace default_cpu(); + +bool is_gpu_place(const Place&); +bool is_cpu_place(const Place&); +bool places_are_same_class(const Place&, const Place&); + +std::ostream& operator<<(std::ostream&, const majel::Place&); + +} // namespace majel diff --git a/paddle/majel/test/CMakeLists.txt b/paddle/majel/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0cc7103b0391184951c3d5c0511ef790e1bcbd2d --- /dev/null +++ b/paddle/majel/test/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB_RECURSE ALL_TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp" "*.cc") + +add_executable(majel_tests ${ALL_TEST_FILES}) +add_dependencies(majel_tests majel) +target_link_libraries(majel_tests + ${Boost_LIBRARIES} + ${GTEST_LIBRARIES} + majel + ) +add_test(majel_tests majel_tests) diff --git a/paddle/majel/test/place_test.cpp b/paddle/majel/test/place_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9a53802b23ef8b225b9e8ef0acfe1b0c5562289 --- /dev/null +++ b/paddle/majel/test/place_test.cpp @@ -0,0 +1,40 @@ +#include "majel/place.h" +#include +#include "gtest/gtest.h" + +TEST(Place, Equality) { + majel::CpuPlace cpu; + majel::GpuPlace g0(0), g1(1), gg0(0); + + EXPECT_EQ(cpu, cpu); + EXPECT_EQ(g0, g0); + EXPECT_EQ(g1, g1); + EXPECT_EQ(g0, gg0); + + EXPECT_NE(g0, g1); + + EXPECT_TRUE(majel::places_are_same_class(g0, gg0)); + EXPECT_FALSE(majel::places_are_same_class(g0, cpu)); +} + +TEST(Place, Default) { + EXPECT_TRUE(majel::is_gpu_place(majel::get_place())); + EXPECT_TRUE(majel::is_gpu_place(majel::default_gpu())); + EXPECT_TRUE(majel::is_cpu_place(majel::default_cpu())); + + majel::set_place(majel::CpuPlace()); + EXPECT_TRUE(majel::is_cpu_place(majel::get_place())); +} + +TEST(Place, Print) { + { + std::stringstream ss; + ss << majel::GpuPlace(1); + EXPECT_EQ("GpuPlace(1)", ss.str()); + } + { + std::stringstream ss; + ss << majel::CpuPlace(); + EXPECT_EQ("CpuPlace", ss.str()); + } +} diff --git a/paddle/majel/test/test_framework.cpp b/paddle/majel/test/test_framework.cpp new file mode 100644 index 0000000000000000000000000000000000000000..443e2dbb3f2b7064f52ccfa017111b7e781f0e97 --- /dev/null +++ b/paddle/majel/test/test_framework.cpp @@ -0,0 +1,6 @@ +#include "gtest/gtest.h" + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 9e72c68d4bb1e3708b01885460b10a0f9dc5afe8..101b44e6c62ecf0b84d65ee7b6e90e64bd7b3272 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -4,9 +4,9 @@ set -xe # Set BASE_IMAGE according to env variables if [ ${WITH_GPU} == "ON" ]; then - BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu14.04" + BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04" else - BASE_IMAGE="ubuntu:14.04" + BASE_IMAGE="ubuntu:16.04" fi DOCKERFILE_GPU_ENV="" @@ -22,6 +22,20 @@ cd /paddle/build # build script will not fail if *.deb does not exist rm *.deb 2>/dev/null || true +cat < /paddle/build/Dockerfile <]" - echo "This script generates doc and doc_cn in the script's directory." - echo "These are common commands used in various situations:" - echo " with_docker build doc and doc_cn with docker" - echo " local build doc and doc_cn locally" -} - - -case "$1" in - "with_docker") - docker run --rm -v $PWD/../../../../:/paddle \ - -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_DOC=ON" paddledev/paddle:dev - ;; - "local") - mkdir -p doc - mkdir -p doc_cn - PADDLE_SOURCE_DIR=$PWD/../../../../ - mkdir -p $PADDLE_SOURCE_DIR/build_doc - pushd $PADDLE_SOURCE_DIR/build_doc - cmake .. -DWITH_DOC=ON - make paddle_docs paddle_docs_cn - popd - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/en/html/* doc - cp -r $PADDLE_SOURCE_DIR/build_doc/doc/cn/html/* doc_cn - rm -rf $PADDLE_SOURCE_DIR/build_doc - ;; - "--help") - usage - ;; - *) - usage - ;; -esac +docker run --rm \ + -v $(git rev-parse --show-toplevel):/paddle \ + -e "WITH_GPU=OFF" \ + -e "WITH_AVX=ON" \ + -e "WITH_DOC=ON" \ + -e "WOBOQ=ON" \ + ${1:-"paddledev/paddle:dev"} diff --git a/paddle/utils/DynamicLoader.cpp b/paddle/utils/DynamicLoader.cpp index 76cf3c300616e6961be905d0e54f3b9fac4922a4..5604a90038b06d2c1a4d9db70e4185cddfd25d3e 100644 --- a/paddle/utils/DynamicLoader.cpp +++ b/paddle/utils/DynamicLoader.cpp @@ -104,20 +104,9 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, CHECK(nullptr != *dso_handle) << "Failed to find dynamic library: " << dlPath << " (" << dlerror() << ") \n" << "Please specify its path correctly using " - "one of the following ways: \n" - - << "Method 1. set cuda and cudnn lib path at " - "runtime. " - << "http://www.paddlepaddle.org/doc/ui/" - "cmd_argument/" - "argument_outline.html \n" - << "For instance, issue command: paddle train " - "--use_gpu=1 " - << "--cuda_dir=/usr/local/cuda/lib64 " - "--cudnn_dir=/usr/local/cudnn/lib " - "...\n" - - << "Method 2. set environment variable " + "following ways: \n" + + << "Method. set environment variable " "LD_LIBRARY_PATH on Linux or " << "DYLD_LIBRARY_PATH on Mac OS. \n" << "For instance, issue command: export " @@ -126,9 +115,7 @@ static inline void GetDsoHandleFromSearchPath(const std::string& search_root, << "Note: After Mac OS 10.11, using the " "DYLD_LIBRARY_PATH is impossible " << "unless System Integrity Protection (SIP) " - "is disabled. However, " - "method 1 " - << "always work well."; + "is disabled."; } void GetCublasDsoHandle(void** dso_handle) { diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 0e752c117c1ecfab72e2da2f830380e9524236e7..a36f0ebfdcb9f90f54ba2d688f9f4bcee2939ef3 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -270,7 +270,7 @@ class CheckWrapper(object): assert isinstance(each, collections.Sequence) for d in each: assert isinstance(d, float) - assert len(each, input_type.dim) + assert len(each) == input_type.dim elif input_type.type == DataType.Index: assert isinstance(each, int) assert each < input_type.dim @@ -304,7 +304,7 @@ class CheckInputTypeWrapper(object): def __call__(self, obj, filename): for items in self.generator(obj, filename): try: - # dict type is required for input_types when item is dict type + # dict type is required for input_types when item is dict type assert (isinstance(items, dict) and \ not isinstance(self.input_types, dict))==False yield items diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 32e31fe2c446fb5d5e2df0819749a60cb8afdfd2..57d30b088b873a94a11483aea536a9e4f6493129 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -138,14 +138,7 @@ def init_config_environment( g_root_submodel=None, g_submodel_map={}, g_submodel_stack=[], - g_add_submodel_suffix=False, - - # Whether current layer needs to pass the image height and width. - # Default value is true, but if it encounters recurrent_layer_group, - # it will be false. The reason is that image is converted to be sequence, - # image height will be sequence length, and image width will be feature - # length of each timestep. - g_pass_height_width=True, ): + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -1437,12 +1430,6 @@ class LayerBase(object): g_current_submodel.layer_names.append(self.config.name) - if self.config.type != 'data' and g_pass_height_width: - height = self.get_input_layer(0).height - width = self.get_input_layer(0).width - if height and width: - self.set_layer_height_width(height, width) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] @@ -3164,8 +3151,6 @@ class WarpCTCLayer(LayerBase): @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): def __init__(self, name, device=None): - global g_pass_height_width - g_pass_height_width = False super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr index 9fae596f281d44dc24c45cb3c750233266e95948..fd5224ca55cd1f642ca2f927f867a7cbf8a47cf6 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr @@ -90,8 +90,6 @@ layers { input_layer_name: "__pool_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 32 - width: 32 } parameters { name: "___conv_0__.w0" diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr index c763a95f9d1aefa022f38e0beef6d1c86ebb360d..03f4f3a31d6c222d949f64341bb8ac4c2a56fc5a 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr @@ -153,8 +153,6 @@ layers { img_size_y: 0 } } - height: 24 - width: 24 } layers { name: "__fc_layer_0__" @@ -165,8 +163,6 @@ layers { input_layer_name: "__block_expand_layer_0__" input_parameter_name: "___fc_layer_0__.w0" } - height: 24 - width: 24 } parameters { name: "___conv_0__.w0" diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index 70c8ec0baa63b9a20506cb786997c0b6d8eedf01..851fe7060fd52120603ebabb4069d67471aa05d0 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -33,11 +33,12 @@ import networks import py_paddle.swig_paddle as api import minibatch import plot +import image __all__ = [ 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', 'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader', - 'topology', 'networks', 'infer', 'plot', 'evaluator' + 'topology', 'networks', 'infer', 'plot', 'evaluator', 'image' ] @@ -52,7 +53,7 @@ def init(**kwargs): args_dict.update(kwargs) # NOTE: overwrite arguments from ENV if it is in kwargs for key in args_dict.keys(): - args.append('--%s=%s' % (key, str(kwargs[key]))) + args.append('--%s=%s' % (key, str(args_dict[key]))) api.initPaddle(*args) diff --git a/python/paddle/v2/dataset/imikolov.py b/python/paddle/v2/dataset/imikolov.py index bf88fe15570682b544bbac802cd65545f756fc3e..dd3a4552d2e1a2b00dde5ddb7ac1d78445bdca51 100644 --- a/python/paddle/v2/dataset/imikolov.py +++ b/python/paddle/v2/dataset/imikolov.py @@ -28,6 +28,11 @@ URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' MD5 = '30177ea32e27c525793142b6bf2c8e2d' +class DataType(object): + NGRAM = 1 + SEQ = 2 + + def word_count(f, word_freq=None): if word_freq is None: word_freq = collections.defaultdict(int) @@ -41,7 +46,7 @@ def word_count(f, word_freq=None): return word_freq -def build_dict(typo_freq=50): +def build_dict(min_word_freq=50): """ Build a word dictionary from the corpus, Keys of the dictionary are words, and values are zero-based IDs of these words. @@ -59,7 +64,7 @@ def build_dict(typo_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = filter(lambda x: x[1] > typo_freq, word_freq.items()) + word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) @@ -69,7 +74,7 @@ def build_dict(typo_freq=50): return word_idx -def reader_creator(filename, word_idx, n): +def reader_creator(filename, word_idx, n, data_type): def reader(): with tarfile.open( paddle.v2.dataset.common.download( @@ -79,16 +84,27 @@ def reader_creator(filename, word_idx, n): UNK = word_idx[''] for l in f: - l = [''] + l.strip().split() + [''] - if len(l) >= n: + if DataType.NGRAM == data_type: + assert n > -1, 'Invalid gram length' + l = [''] + l.strip().split() + [''] + if len(l) >= n: + l = [word_idx.get(w, UNK) for w in l] + for i in range(n, len(l) + 1): + yield tuple(l[i - n:i]) + elif DataType.SEQ == data_type: + l = l.strip().split() l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): - yield tuple(l[i - n:i]) + src_seq = [word_idx['']] + l + trg_seq = l + [word_idx['']] + if n > 0 and len(src_seq) > n: continue + yield src_seq, trg_seq + else: + assert False, 'Unknow data type' return reader -def train(word_idx, n): +def train(word_idx, n, data_type=DataType.NGRAM): """ imikolov training set creator. @@ -97,15 +113,18 @@ def train(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Training reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.train.txt', word_idx, n, + data_type) -def test(word_idx, n): +def test(word_idx, n, data_type=DataType.NGRAM): """ imikolov test set creator. @@ -114,12 +133,15 @@ def test(word_idx, n): :param word_idx: word dictionary :type word_idx: dict - :param n: sliding window size + :param n: sliding window size if type is ngram, otherwise max length of sequence :type n: int + :param data_type: data type (ngram or sequence) + :type data_type: member variable of DataType (NGRAM or SEQ) :return: Test reader creator :rtype: callable """ - return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n) + return reader_creator('./simple-examples/data/ptb.valid.txt', word_idx, n, + data_type) def fetch(): diff --git a/python/paddle/v2/dataset/tests/imikolov_test.py b/python/paddle/v2/dataset/tests/imikolov_test.py index 009e55243a594e5e235c36fb0223ec70754d17f3..4e52810e6b924e0796e3d836dbbcb27ede2c9e25 100644 --- a/python/paddle/v2/dataset/tests/imikolov_test.py +++ b/python/paddle/v2/dataset/tests/imikolov_test.py @@ -13,10 +13,37 @@ class TestMikolov(unittest.TestCase): n = 5 self.check_reader(paddle.v2.dataset.imikolov.train(WORD_DICT, n), n) + first_line = 'aer banknote berlitz calloway centrust cluett fromstein '\ + 'gitano guterman hydro-quebec ipo kia memotec mlx nahb punts '\ + 'rake regatta rubens sim snack-food ssangyong swapo wachter' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.train( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_test(self): n = 5 self.check_reader(paddle.v2.dataset.imikolov.test(WORD_DICT, n), n) + first_line = 'consumers may want to move their telephones a little '\ + 'closer to the tv set' + first_line = [ + WORD_DICT.get(ch, WORD_DICT['']) + for ch in first_line.split(' ') + ] + for l in paddle.v2.dataset.imikolov.test( + WORD_DICT, n=-1, + data_type=paddle.v2.dataset.imikolov.DataType.SEQ)(): + read_line = l[0][1:] + break + self.assertEqual(first_line, read_line) + def test_total(self): _, idx = zip(*WORD_DICT.items()) self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py new file mode 100644 index 0000000000000000000000000000000000000000..13f53919aa49694f722d4bf20a7d01af3e3e6533 --- /dev/null +++ b/python/paddle/v2/image.py @@ -0,0 +1,236 @@ +import numpy as np +try: + import cv2 +except ImportError: + cv2 = None + +from cv2 import resize + +__all__ = [ + "load_image", "resize_short", "to_chw", "center_crop", "random_crop", + "left_right_flip", "simple_transform", "load_and_transform" +] +""" +This file contains some common interfaces for image preprocess. +Many users are confused about the image layout. We introduce +the image layout as follows. + +- CHW Layout + - The abbreviations: C=channel, H=Height, W=Width + - The default layout of image opened by cv2 or PIL is HWC. + PaddlePaddle only supports the CHW layout. And CHW is simply + a transpose of HWC. It must transpose the input image. + +- Color format: RGB or BGR + OpenCV use BGR color format. PIL use RGB color format. Both + formats can be used for training. Noted that, the format should + be keep consistent between the training and inference peroid. +""" + + +def load_image(file, is_color=True): + """ + Load an color or gray image from the file path. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + + :param file: the input image path. + :type file: string + :param is_color: If set is_color True, it will load and + return a color image. Otherwise, it will + load and return a gray image. + """ + # cv2.IMAGE_COLOR for OpenCV3 + # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version + # cv2.IMAGE_GRAYSCALE for OpenCV3 + # cv2.CV_LOAD_IMAGE_GRAYSCALE for older OpenCV Version + # Here, use constant 1 and 0 + # 1: COLOR, 0: GRAYSCALE + flag = 1 if is_color else 0 + im = cv2.imread(file, flag) + return im + + +def resize_short(im, size): + """ + Resize an image so that the length of shorter edge is size. + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the shorter edge size of image after resizing. + :type size: int + """ + assert im.shape[-1] == 1 or im.shape[-1] == 3 + h, w = im.shape[:2] + h_new, w_new = size, size + if h > w: + h_new = size * h / w + else: + w_new = size * w / h + im = resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) + return im + + +def to_chw(im, order=(2, 0, 1)): + """ + Transpose the input image order. The image layout is HWC format + opened by cv2 or PIL. Transpose the input image to CHW layout + according the order (2,0,1). + + Example usage: + + .. code-block:: python + im = load_image('cat.jpg') + im = resize_short(im, 256) + im = to_chw(im) + + :param im: the input image with HWC layout. + :type im: ndarray + :param order: the transposed order. + :type order: tuple|list + """ + assert len(im.shape) == len(order) + im = im.transpose(order) + return im + + +def center_crop(im, size, is_color=True): + """ + Crop the center of image with size. + + Example usage: + + .. code-block:: python + im = center_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size. + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = (h - size) / 2 + w_start = (w - size) / 2 + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def random_crop(im, size, is_color=True): + """ + Randomly crop input image with size. + + Example usage: + + .. code-block:: python + im = random_crop(im, 224) + + :param im: the input image with HWC layout. + :type im: ndarray + :param size: the cropping size. + :type size: int + :param is_color: whether the image is color or not. + :type is_color: bool + """ + h, w = im.shape[:2] + h_start = np.random.randint(0, h - size + 1) + w_start = np.random.randint(0, w - size + 1) + h_end, w_end = h_start + size, w_start + size + if is_color: + im = im[h_start:h_end, w_start:w_end, :] + else: + im = im[h_start:h_end, w_start:w_end] + return im + + +def left_right_flip(im): + """ + Flip an image along the horizontal direction. + Return the flipped image. + + Example usage: + + .. code-block:: python + im = left_right_flip(im) + + :paam im: input image with HWC layout + :type im: ndarray + """ + if len(im.shape) == 3: + return im[:, ::-1, :] + else: + return im[:, ::-1, :] + + +def simple_transform(im, resize_size, crop_size, is_train, is_color=True): + """ + Simply data argumentation for training. These operations include + resizing, croping and flipping. + + Example usage: + + .. code-block:: python + im = simple_transform(im, 256, 224, True) + + :param im: The input image with HWC layout. + :type im: ndarray + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = resize_short(im, resize_size) + if is_train: + im = random_crop(im, crop_size) + if np.random.randint(2) == 0: + im = left_right_flip(im) + else: + im = center_crop(im, crop_size) + im = to_chw(im) + + return im + + +def load_and_transform(filename, + resize_size, + crop_size, + is_train, + is_color=True): + """ + Load image from the input file `filename` and transform image for + data argumentation. Please refer to the `simple_transform` interface + for the transform operations. + + Example usage: + + .. code-block:: python + im = load_and_transform('cat.jpg', 256, 224, True) + + :param filename: The file name of input image. + :type filename: string + :param resize_size: The shorter edge length of the resized image. + :type resize_size: int + :param crop_size: The cropping size. + :type crop_size: int + :param is_train: Whether it is training or not. + :type is_train: bool + """ + im = load_image(filename) + im = simple_transform(im, resize_size, crop_size, is_train, is_color) + return im diff --git a/python/paddle/v2/tests/CMakeLists.txt b/python/paddle/v2/tests/CMakeLists.txt index 5554a37df03cb31b37e9bbf68faa606d7c2cadde..eb02e53706b4834eb9dc75d0e3a809772b124725 100644 --- a/python/paddle/v2/tests/CMakeLists.txt +++ b/python/paddle/v2/tests/CMakeLists.txt @@ -1 +1,2 @@ -add_python_test(test_v2_api test_data_feeder.py test_parameters.py test_layer.py test_rnn_layer.py test_topology.py) +add_python_test(test_v2_api test_data_feeder.py test_parameters.py +test_layer.py test_rnn_layer.py test_topology.py test_image.py) diff --git a/python/paddle/v2/tests/cat.jpg b/python/paddle/v2/tests/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bc1fbbd371216b9904b522ed302700c79d2e4876 Binary files /dev/null and b/python/paddle/v2/tests/cat.jpg differ diff --git a/python/paddle/v2/tests/test_image.py b/python/paddle/v2/tests/test_image.py new file mode 100644 index 0000000000000000000000000000000000000000..b2d773510de28ca2614e95b465c73b82aa7b0463 --- /dev/null +++ b/python/paddle/v2/tests/test_image.py @@ -0,0 +1,42 @@ +# Copyright PaddlePaddle contributors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import numpy as np + +import paddle.v2.image as image + + +class Image(unittest.TestCase): + def test_resize_flip_chw(self): + # resize + im = image.load_image('cat.jpg') + im = image.resize_short(im, 256) + self.assertEqual(256, min(im.shape[:2])) + self.assertEqual(3, im.shape[2]) + + # flip + im = image.left_right_flip(im) + im2 = np.flip(im, 1) + self.assertEqual(im.all(), im2.all()) + + # to_chw + h, w, c = im.shape + im = image.to_chw(im) + self.assertEqual(c, im.shape[0]) + self.assertEqual(h, im.shape[1]) + self.assertEqual(w, im.shape[2]) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/setup.py.in b/python/setup.py.in index 5dfb46192ae54fdc36b0867312cf156aefb84f84..7d9438e3f8132c2a7fa4774750f5fd15f3beed14 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,6 +18,7 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", + "opencv-python", ], packages=packages, package_dir={