diff --git a/README.md b/README.md index b9793c3eab5d40c28f01cc67ad607b97261b3235..db0fbd88b250cdc2a3cc77521cc1c2cea77c6e87 100644 --- a/README.md +++ b/README.md @@ -51,19 +51,19 @@ Please refer to our [release announcement](https://github.com/PaddlePaddle/Paddl - **Connected to Products** In addition, PaddlePaddle is also designed to be easily deployable. At Baidu, - PaddlePaddle has been deployed into products or service with a vast number + PaddlePaddle has been deployed into products and services with a vast number of users, including ad click-through rate (CTR) prediction, large-scale image classification, optical character recognition(OCR), search ranking, computer virus detection, recommendation, etc. It is widely utilized in products at - Baidu and it has achieved a significant impact. We hope you can also exploit - the capability of PaddlePaddle to make a huge impact for your product. + Baidu and it has achieved a significant impact. We hope you can also explore + the capability of PaddlePaddle to make an impact on your product. ## Installation It is recommended to check out the [Docker installation guide](http://doc.paddlepaddle.org/develop/doc/getstarted/build_and_install/docker_install_en.html) before looking into the -[build from source guide](http://doc.paddlepaddle.org/develop/doc/getstarted/build_and_install/build_from_source_en.html) +[build from source guide](http://doc.paddlepaddle.org/develop/doc/getstarted/build_and_install/build_from_source_en.html). ## Documentation @@ -72,7 +72,7 @@ We provide [English](http://doc.paddlepaddle.org/develop/doc/) and - [Deep Learning 101](http://book.paddlepaddle.org/index.html) - You might want to start from this online interactive book that can run in Jupyter Notebook. + You might want to start from this online interactive book that can run in a Jupyter Notebook. - [Distributed Training](http://doc.paddlepaddle.org/develop/doc/howto/usage/cluster/cluster_train_en.html) diff --git a/benchmark/paddle/image/run_mkldnn.sh b/benchmark/paddle/image/run_mkldnn.sh index b6cd6fe03b381d2b6529116f934ce7ce03d63546..e31fec1cd850157d90ddcab2d559d52381ecd317 100755 --- a/benchmark/paddle/image/run_mkldnn.sh +++ b/benchmark/paddle/image/run_mkldnn.sh @@ -1,10 +1,9 @@ set -e -unset OMP_NUM_THREADS MKL_NUM_THREADS -export OMP_DYNAMIC="FALSE" -export KMP_AFFINITY="granularity=fine,compact,0,0" - function train() { + unset OMP_NUM_THREADS MKL_NUM_THREADS + export OMP_DYNAMIC="FALSE" + export KMP_AFFINITY="granularity=fine,compact,0,0" topology=$1 bs=$2 use_mkldnn=$3 @@ -13,9 +12,13 @@ function train() { log="logs/${topology}-mkldnn-${bs}.log" elif [ $3 == "False" ]; then thread=`nproc` + # each trainer_count use only 1 core to avoid conflict + export OMP_NUM_THREADS=1 + export MKL_NUM_THREADS=1 log="logs/${topology}-${thread}mklml-${bs}.log" else echo "Wrong input $3, use True or False." + exit 0 fi args="batch_size=${bs}" config="${topology}.py" diff --git a/doc/design/tensor_array.md b/doc/design/tensor_array.md new file mode 100644 index 0000000000000000000000000000000000000000..a0419ec002159893b035fae1300fce489e68936a --- /dev/null +++ b/doc/design/tensor_array.md @@ -0,0 +1,73 @@ +# Design for TensorArray +TensorArray as a new concept is borrowed from TensorFlow, +it is meant to be used with dynamic iteration primitives such as `while_loop` and `map_fn`. + +This concept can be used to support our new design of dynamic operations, and help to refactor some existing variant-sentence-related layers, +such as `RecurrentGradientMachine`. + +In [our design for dynamic RNN](https://github.com/PaddlePaddle/Paddle/pull/4401), +`TensorArray` is used to segment inputs and store states in all time steps. +By providing some methods similar to a C++ array, +the definition of some state-based dynamic models such as RNN could be more natural and highly flexible. + +## Dynamic-Related Methods +Some basic methods should be proposed as follows: + +### stack() +Pack the values in a `TensorArray` into a tensor with rank one higher than each tensor in `values`. +### unstack(axis=0) +Unpacks the given dimension of a rank-`R` tensor into rank-`(R-1)` tensors. +### concat() +Return the values in the `TensorArray` as a concatenated Tensor. +### write(index, value, data_shared=true) +Write value into index of the TensorArray. +### read(index) +Read the value at location `index` in the `TensorArray`. +### size() +Return the number of values. + +## LoDTensor-related Supports +The `RecurrentGradientMachine` in Paddle serves as a flexible RNN layer; it takes variant length sequences as input, +because each step of RNN could only take a tensor-represented batch of data as input, +some preprocess should be taken on the inputs such as sorting the sentences by their length in descending order and cut each word and pack to new batches. + +Such cut-like operations can be embedded into `TensorArray` as general methods called `unpack` and `pack`. + +With these two methods, a variant-sentence-RNN can be implemented like + +```c++ +// input is the varient-length data +LodTensor sentence_input(xxx); +TensorArray ta; +Tensor indice_map; +Tensor boot_state = xxx; // to initialize rnn's first state +TensorArray::unpack(input, 1/*level*/, true/*sort_by_length*/, &ta, &indice_map); +TessorArray step_outputs; +TensorArray states; + +for (int step = 0; step = ta.size(); step++) { + auto state = states.read(step); + // rnnstep is a function which acts like a step of RNN + auto step_input = ta.read(step); + auto step_output = rnnstep(step_input, state); + step_outputs.write(step_output, true/*data_shared*/); +} + +// rnn_output is the final output of an rnn +LoDTensor rnn_output = ta.pack(ta, indice_map); +``` +the code above shows that by embedding the LoDTensor-related preprocess operations into `TensorArray`, +the implementation of a RNN that supports varient-length sentences is far more concise than `RecurrentGradientMachine` because the latter mixes all the codes together, hard to read and extend. + + +some details are as follows. + +### unpack(level, sort_by_length) +Split LodTensor in some `level` and generate batches, if set `sort_by_length`, will sort by length. + +Returns: + +- a new `TensorArray`, whose values are LodTensors and represents batches of data. +- an int32 Tensor, which stores the map from the new batch's indices to original LoDTensor +### pack(level, indices_map) +Recover the original LoD-arranged LoDTensor with the values in a `TensorArray` and `level` and `indices_map`. diff --git a/doc/faq/build_and_install/index_cn.rst b/doc/faq/build_and_install/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..f1677e216f31d79b53ac29a0afbf6fbb886a0dcd --- /dev/null +++ b/doc/faq/build_and_install/index_cn.rst @@ -0,0 +1,111 @@ +################### +编译安装与单元测试 +################### + +.. contents:: + +1. 运行Docker GPU镜像出现 "CUDA driver version is insufficient" +---------------------------------------------------------------- + +用户在使用PaddlePaddle GPU的Docker镜像的时候，常常出现 `Cuda Error: CUDA driver version is insufficient for CUDA runtime version`, 原因在于没有把机器上CUDA相关的驱动和库映射到容器内部。 +具体的解决方法是： + +.. code-block:: bash + + $ export CUDA_SO="$(\ls usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" + $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') + $ docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddlepaddle:latest-gpu + +更多关于Docker的安装与使用, 请参考 `PaddlePaddle Docker 文档 `_ 。 + + +2. CMake源码编译, 找到的PythonLibs和PythonInterp版本不一致 +---------------------------------------------------------------- + +这是目前CMake寻找Python的逻辑存在缺陷，如果系统安装了多个Python版本，CMake找到的Python库和Python解释器版本可能有不一致现象，导致编译PaddlePaddle失败。正确的解决方法是， +用户强制指定特定的Python版本，具体操作如下： + + .. code-block:: bash + + cmake .. -DPYTHON_EXECUTABLE= -DPYTHON_LIBRARY= -DPYTHON_INCLUDE_DIR= + +用户需要指定本机上Python的路径：````, ````, ```` + +3. CMake源码编译，Paddle版本号为0.0.0 +-------------------------------------- + +如果运行 :code:`paddle version`, 出现 :code:`PaddlePaddle 0.0.0`；或者运行 :code:`cmake ..`，出现 + +.. code-block:: bash + + CMake Warning at cmake/version.cmake:20 (message): + Cannot add paddle version from git tag + +那么用户需要拉取所有的远程分支到本机，命令为 :code:`git fetch upstream`，然后重新cmake即可。 + +4. paddlepaddle\*.whl is not a supported wheel on this platform. +------------------------------------------------------------------------ + +出现这个问题的主要原因是，没有找到和当前系统匹配的paddlepaddle安装包。最新的paddlepaddle python安装包支持Linux x86_64和MacOS 10.12操作系统，并安装了python 2.7和pip 9.0.1。 + +更新 :code:`pip` 包的方法是\: + +.. code-block:: bash + + pip install --upgrade pip + +如果还不行，可以执行 :code:`python -c "import pip; print(pip.pep425tags.get_supported())"` 获取当前系统支持的python包的后缀， +并对比是否和正在安装的后缀一致。 + +如果系统支持的是 :code:`linux_x86_64` 而安装包是 :code:`manylinux1_x86_64` ，需要升级pip版本到最新； +如果系统支持 :code:`manylinux1_x86_64` 而安装包（本地）是 :code:`linux_x86_64` ，可以重命名这个whl包为 :code:`manylinux1_x86_64` 再安装。 + +5. 编译安装后执行 import paddle.v2 as paddle 报ImportError: No module named v2 +------------------------------------------------------------------------------------------ +先查看一下是否曾经安装过paddle v1版本，有的话需要先卸载： + +pip uninstall py_paddle paddle + +然后安装paddle的python环境, 在build目录下执行 + +pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl + +6. 遇到“非法指令”或者是“illegal instruction” +-------------------------------------------- + +PaddlePaddle使用avx SIMD指令提高cpu执行效率，因此错误的使用二进制发行版可能会导致这种错误，请选择正确的版本。 + +7. python相关的单元测试都过不了 +-------------------------------- + +如果出现以下python相关的单元测试都过不了的情况： + +.. code-block:: bash + + 24 - test_PyDataProvider (Failed) + 26 - test_RecurrentGradientMachine (Failed) + 27 - test_NetworkCompare (Failed) + 28 - test_PyDataProvider2 (Failed) + 32 - test_Prediction (Failed) + 33 - test_Compare (Failed) + 34 - test_Trainer (Failed) + 35 - test_TrainerOnePass (Failed) + 36 - test_CompareTwoNets (Failed) + 37 - test_CompareTwoOpts (Failed) + 38 - test_CompareSparse (Failed) + 39 - test_recurrent_machine_generation (Failed) + 40 - test_PyDataProviderWrapper (Failed) + 41 - test_config_parser (Failed) + 42 - test_swig_api (Failed) + 43 - layers_test (Failed) + +并且查询PaddlePaddle单元测试的日志，提示： + +.. code-block:: bash + + paddle package is already in your PYTHONPATH. But unittest need a clean environment. + Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'. + +解决办法是： + +* 卸载PaddlePaddle包 :code:`pip uninstall paddle`, 清理掉老旧的PaddlePaddle安装包，使得单元测试有一个干净的环境。如果PaddlePaddle包已经在python的site-packages里面，单元测试会引用site-packages里面的python包，而不是源码目录里 :code:`/python` 目录下的python包。同时，即便设置 :code:`PYTHONPATH` 到 :code:`/python` 也没用，因为python的搜索路径是优先已经安装的python包。 diff --git a/doc/faq/cluster/index_cn.rst b/doc/faq/cluster/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..e59c1e1a54a0c876d1e6e89f88030de59fb9fc1a --- /dev/null +++ b/doc/faq/cluster/index_cn.rst @@ -0,0 +1,17 @@ +############### +集群训练与预测 +############### + +.. contents:: + +1. 集群多节点训练，日志中保存均为网络通信类错误 +------------------------------------------------ + +集群多节点训练，日志报错为网络通信类错误，比如 :code:`Connection reset by peer` 等。 +此类报错通常是由于某一个节点的错误导致这个节点的训练进程退出，从而引发其他节点无法连接导致，可以参考下面的步骤排查： + +* 从 :code:`train.log` ， :code:`server.log` 找到最早报错的地方，查看是否是其他错误引发的报错（比如FPE，内存不足，磁盘空间不足等）。 + +* 如果发现最早的报错就是网络通信的问题，很有可能是非独占方式执行导致的端口冲突，可以联系OP，看当前MPI集群是否支持resource=full参数提交，如果支持增加此参数提交，并更换job 端口。 + +* 如果当前MPI集群并不支持任务独占模式，可以联系OP是否可以更换集群或升级当前集群。 diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index d69d111917ca7a79bc65b051c8eefaba165d77bd..9929767cac212237b3e2c3a547ba9a3c9d5f0979 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -1,592 +1,11 @@ -#################### FAQ -#################### +==== -.. contents:: +.. toctree:: + :maxdepth: 1 -1. 如何减少内存占用 ---------------------------------- - -神经网络的训练本身是一个非常消耗内存和显存的工作，经常会消耗数10GB的内存和数GB的显存。 -PaddlePaddle的内存占用主要分为如下几个方面\: - -* DataProvider缓冲池内存（只针对内存） -* 神经元激活内存（针对内存和显存） -* 参数内存 （针对内存和显存） -* 其他内存杂项 - -其中，其他内存杂项是指PaddlePaddle本身所用的一些内存，包括字符串分配，临时变量等等，暂不考虑在内。 - -减少DataProvider缓冲池内存 -++++++++++++++++++++++++++ - -PyDataProvider使用的是异步加载，同时在内存里直接随即选取数据来做Shuffle。即 - -.. graphviz:: - - digraph { - rankdir=LR; - 数据文件 -> 内存池 -> PaddlePaddle训练 - } - -所以，减小这个内存池即可减小内存占用，同时也可以加速开始训练前数据载入的过程。但是，这 -个内存池实际上决定了shuffle的粒度。所以，如果将这个内存池减小，又要保证数据是随机的， -那么最好将数据文件在每次读取之前做一次shuffle。可能的代码为 - -.. literalinclude:: src/reduce_min_pool_size.py - -这样做可以极大的减少内存占用，并且可能会加速训练过程，详细文档参考 :ref:`api_pydataprovider2` 。 - -神经元激活内存 -++++++++++++++ - -神经网络在训练的时候，会对每一个激活暂存一些数据，如神经元激活值等。 -在反向传递的时候，这些数据会被用来更新参数。这些数据使用的内存主要和两个参数有关系， -一是batch size，另一个是每条序列(Sequence)长度。所以，其实也是和每个mini-batch中包含 -的时间步信息成正比。 - -所以做法可以有两种： - -* 减小batch size。 即在网络配置中 :code:`settings(batch_size=1000)` 设置成一个小一些的值。但是batch size本身是神经网络的超参数，减小batch size可能会对训练结果产生影响。 -* 减小序列的长度，或者直接扔掉非常长的序列。比如，一个数据集大部分序列长度是100-200, - 但是突然有一个10000长的序列，就很容易导致内存超限，特别是在LSTM等RNN中。 - -参数内存 -++++++++ - -PaddlePaddle支持非常多的优化算法(Optimizer)，不同的优化算法需要使用不同大小的内存。 -例如使用 :code:`adadelta` 算法，则需要使用等于权重参数规模大约5倍的内存。举例，如果参数保存下来的模型目录 -文件为 :code:`100M`， 那么该优化算法至少需要 :code:`500M` 的内存。 - -可以考虑使用一些优化算法，例如 :code:`momentum`。 - -2. 如何加速PaddlePaddle的训练速度 ---------------------------------- - -加速PaddlePaddle训练可以考虑从以下几个方面\： - -* 减少数据载入的耗时 -* 加速训练速度 -* 利用分布式训练驾驭更多的计算资源 - -减少数据载入的耗时 -++++++++++++++++++ - -使用\ :code:`pydataprovider`\ 时，可以减少缓存池的大小，同时设置内存缓存功能，即可以极大的加速数据载入流程。 -:code:`DataProvider` 缓存池的减小，和之前减小通过减小缓存池来减小内存占用的原理一致。 - -.. literalinclude:: src/reduce_min_pool_size.py - -同时 :code:`@provider` 接口有一个 :code:`cache` 参数来控制缓存方法，将其设置成 :code:`CacheType.CACHE_PASS_IN_MEM` 的话，会将第一个 :code:`pass` (过完所有训练数据即为一个pass)生成的数据缓存在内存里，在之后的 :code:`pass` 中，不会再从 :code:`python` 端读取数据，而是直接从内存的缓存里读取数据。这也会极大减少数据读入的耗时。 - - -加速训练速度 -++++++++++++ - -PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`sparse_binary_vector` 、 :code:`sparse_vector` 、或者 :code:`integer_value` 的任一一种。同时，与这个训练数据交互的Layer，需要将其Parameter设置成 sparse 更新模式，即设置 :code:`sparse_update=True` - -这里使用简单的 :code:`word2vec` 训练语言模型距离，具体使用方法为\: - -使用一个词前两个词和后两个词，来预测这个中间的词。这个任务的DataProvider为\: - -.. literalinclude:: src/word2vec_dataprovider.py - -这个任务的配置为\: - -.. literalinclude:: src/word2vec_config.py - - -利用更多的计算资源 -++++++++++++++++++ - -利用更多的计算资源可以分为一下几个方式来进行\: - -* 单机CPU训练 - - * 使用多线程训练。设置命令行参数 :code:`trainer_count`。 - -* 单机GPU训练 - - * 使用显卡训练。设置命令行参数 :code:`use_gpu`。 - * 使用多块显卡训练。设置命令行参数 :code:`use_gpu` 和 :code:`trainer_count` 。 - -* 多机训练 - - * 请参考 :ref:`cluster_train` 。 - - -3. 遇到“非法指令”或者是“illegal instruction” --------------------------------------------- - -PaddlePaddle使用avx SIMD指令提高cpu执行效率，因此错误的使用二进制发行版可能会导致这种错误，请选择正确的版本。 - -4. 如何选择SGD算法的学习率 --------------------------- - -在采用sgd/async_sgd进行训练时，一个重要的问题是选择正确的learning_rate。如果learning_rate太大，那么训练有可能不收敛，如果learning_rate太小，那么收敛可能很慢，导致训练时间过长。 - -通常做法是从一个比较大的learning_rate开始试，如果不收敛，那减少学习率10倍继续试验，直到训练收敛为止。那么如何判断训练不收敛呢？可以估计出如果模型采用不变的输出最小的cost0是多少。 - -如果训练过程的的cost明显高于这个常数输出的cost，那么我们可以判断为训练不收敛。举一个例子，假如我们是三分类问题，采用multi-class-cross-entropy作为cost，数据中0,1,2三类的比例为 :code:`0.2, 0.5, 0.3` , 那么常数输出所能达到的最小cost是 :code:`-(0.2*log(0.2)+0.5*log(0.5)+0.3*log(0.3))=1.03` 。如果训练一个pass（或者更早）后，cost还大于这个数，那么可以认为训练不收敛，应该降低学习率。 - - -5. 如何初始化参数 ------------------ - -默认情况下，PaddlePaddle使用均值0，标准差为 :math:`\frac{1}{\sqrt{d}}` 来初始化参数。其中 :math:`d` 为参数矩阵的宽度。这种初始化方式在一般情况下不会产生很差的结果。如果用户想要自定义初始化方式，PaddlePaddle目前提供两种参数初始化的方式\: - -* 高斯分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_mean=0.0, initial_std=1.0)` -* 均匀分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0)` - -比如设置一个全连接层的参数初始化方式和bias初始化方式，可以使用如下代码。 - -.. code-block:: python - - hidden = fc_layer(input=ipt, param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0), - bias_attr=ParamAttr(initial_mean=1.0, initial_std=0.0)) - -上述代码将bias全部初始化为1.0, 同时将参数初始化为 :code:`[1.0, -1.0]` 的均匀分布。 - -6. 如何共享参数 ---------------- - -PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字的参数，会共享参数。设置参数的名字，可以使用 :code:`ParamAttr(name="YOUR_PARAM_NAME")` 来设置。更方便的设置方式，是使得要共享的参数使用同样的 :code:`ParamAttr` 对象。 - -简单的全连接网络，参数共享的配置示例为\: - -.. literalinclude:: ../../python/paddle/trainer_config_helpers/tests/configs/shared_fc.py - -这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 - -7. paddlepaddle\*.whl is not a supported wheel on this platform. ------------------------------------------------------------------------- - -出现这个问题的主要原因是，没有找到和当前系统匹配的paddlepaddle安装包。最新的paddlepaddle python安装包支持Linux x86_64和MacOS 10.12操作系统，并安装了python 2.7和pip 9.0.1。 - -更新 :code:`pip` 包的方法是\: - -.. code-block:: bash - - pip install --upgrade pip - -如果还不行，可以执行 :code:`python -c "import pip; print(pip.pep425tags.get_supported())"` 获取当前系统支持的python包的后缀， -并对比是否和正在安装的后缀一致。 - -如果系统支持的是 :code:`linux_x86_64` 而安装包是 :code:`manylinux1_x86_64` ，需要升级pip版本到最新； -如果系统支持 :code:`manylinux1_x86_64` 而安装包（本地）是 :code:`linux_x86_64` ，可以重命名这个whl包为 :code:`manylinux1_x86_64` 再安装。 - -8. python相关的单元测试都过不了 --------------------------------- - -如果出现以下python相关的单元测试都过不了的情况： - -.. code-block:: bash - - 24 - test_PyDataProvider (Failed) - 26 - test_RecurrentGradientMachine (Failed) - 27 - test_NetworkCompare (Failed) - 28 - test_PyDataProvider2 (Failed) - 32 - test_Prediction (Failed) - 33 - test_Compare (Failed) - 34 - test_Trainer (Failed) - 35 - test_TrainerOnePass (Failed) - 36 - test_CompareTwoNets (Failed) - 37 - test_CompareTwoOpts (Failed) - 38 - test_CompareSparse (Failed) - 39 - test_recurrent_machine_generation (Failed) - 40 - test_PyDataProviderWrapper (Failed) - 41 - test_config_parser (Failed) - 42 - test_swig_api (Failed) - 43 - layers_test (Failed) - -并且查询PaddlePaddle单元测试的日志，提示： - -.. code-block:: bash - - paddle package is already in your PYTHONPATH. But unittest need a clean environment. - Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'. - -解决办法是： - -* 卸载PaddlePaddle包 :code:`pip uninstall paddle`, 清理掉老旧的PaddlePaddle安装包，使得单元测试有一个干净的环境。如果PaddlePaddle包已经在python的site-packages里面，单元测试会引用site-packages里面的python包，而不是源码目录里 :code:`/python` 目录下的python包。同时，即便设置 :code:`PYTHONPATH` 到 :code:`/python` 也没用，因为python的搜索路径是优先已经安装的python包。 - - -9. 运行Docker GPU镜像出现 "CUDA driver version is insufficient" ----------------------------------------------------------------- - -用户在使用PaddlePaddle GPU的Docker镜像的时候，常常出现 `Cuda Error: CUDA driver version is insufficient for CUDA runtime version`, 原因在于没有把机器上CUDA相关的驱动和库映射到容器内部。 -具体的解决方法是： - -.. code-block:: bash - - $ export CUDA_SO="$(\ls usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" - $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - $ docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddlepaddle:latest-gpu - -更多关于Docker的安装与使用, 请参考 `PaddlePaddle Docker 文档 `_ 。 - - -10. CMake源码编译, 找到的PythonLibs和PythonInterp版本不一致 ----------------------------------------------------------------- - -这是目前CMake寻找Python的逻辑存在缺陷，如果系统安装了多个Python版本，CMake找到的Python库和Python解释器版本可能有不一致现象，导致编译PaddlePaddle失败。正确的解决方法是， -用户强制指定特定的Python版本，具体操作如下： - - .. code-block:: bash - - cmake .. -DPYTHON_EXECUTABLE= -DPYTHON_LIBRARY= -DPYTHON_INCLUDE_DIR= - -用户需要指定本机上Python的路径：````, ````, ```` - -11. CMake源码编译，Paddle版本号为0.0.0 --------------------------------------- - -如果运行 :code:`paddle version`, 出现 :code:`PaddlePaddle 0.0.0`；或者运行 :code:`cmake ..`，出现 - -.. code-block:: bash - - CMake Warning at cmake/version.cmake:20 (message): - Cannot add paddle version from git tag - -那么用户需要拉取所有的远程分支到本机，命令为 :code:`git fetch upstream`，然后重新cmake即可。 - -12. A protocol message was rejected because it was too big ------------------------------------------------------------- - -如果在训练NLP相关模型时，出现以下错误： - -.. code-block:: bash - - [libprotobuf ERROR google/protobuf/io/coded_stream.cc:171] A protocol message was rejected because it was too big (more than 67108864 bytes). To increase the limit (or to disable these warnings), see CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h. - F1205 14:59:50.295174 14703 TrainerConfigHelper.cpp:59] Check failed: m->conf.ParseFromString(configProtoStr) - -可能的原因是：传给dataprovider的某一个args过大，一般是由于直接传递大字典导致的。错误的define_py_data_sources2类似： - -.. code-block:: python - - src_dict = dict() - for line_count, line in enumerate(open(src_dict_path, "r")): - src_dict[line.strip()] = line_count - - define_py_data_sources2( - train_list, - test_list, - module="dataprovider", - obj="process", - args={"src_dict": src_dict}) - -解决方案是：将字典的地址作为args传给dataprovider，然后在dataprovider里面根据该地址加载字典。即define_py_data_sources2应改为： - -.. code-block:: python - - define_py_data_sources2( - train_list, - test_list, - module="dataprovider", - obj="process", - args={"src_dict_path": src_dict_path}) - -完整源码可参考 `seqToseq `_ 示例。 - -13. 如何指定GPU设备 -------------------- - -例如机器上有4块GPU，编号从0开始，指定使用2、3号GPU： - -* 方式1：通过 `CUDA_VISIBLE_DEVICES `_ 环境变量来指定特定的GPU。 - -.. code-block:: bash - - env CUDA_VISIBLE_DEVICES=2,3 paddle train --use_gpu=true --trainer_count=2 - -* 方式2：通过命令行参数 ``--gpu_id`` 指定。 - -.. code-block:: bash - - paddle train --use_gpu=true --trainer_count=2 --gpu_id=2 - - -14. 训练过程中出现 :code:`Floating point exception`, 训练因此退出怎么办? ------------------------------------------------------------------------- - -Paddle二进制在运行时捕获了浮点数异常，只要出现浮点数异常(即训练过程中出现NaN或者Inf)，立刻退出。浮点异常通常的原因是浮点数溢出、除零等问题。 -主要原因包括两个方面: - -* 训练过程中参数或者训练过程中的梯度尺度过大，导致参数累加，乘除等时候，导致了浮点数溢出。 -* 模型一直不收敛，发散到了一个数值特别大的地方。 -* 训练数据有问题，导致参数收敛到了一些奇异的情况。或者输入数据尺度过大，有些特征的取值达到数百万，这时进行矩阵乘法运算就可能导致浮点数溢出。 - -这里有两种有效的解决方法： - -1. 设置 :code:`gradient_clipping_threshold` 参数，示例代码如下： - -.. code-block:: python - -optimizer = paddle.optimizer.RMSProp( - learning_rate=1e-3, - gradient_clipping_threshold=10.0, - regularization=paddle.optimizer.L2Regularization(rate=8e-4)) - -具体可以参考 `nmt_without_attention `_ 示例。 - -2. 设置 :code:`error_clipping_threshold` 参数，示例代码如下： - -.. code-block:: python - -decoder_inputs = paddle.layer.fc( - act=paddle.activation.Linear(), - size=decoder_size * 3, - bias_attr=False, - input=[context, current_word], - layer_attr=paddle.attr.ExtraLayerAttribute( - error_clipping_threshold=100.0)) - -完整代码可以参考示例 `machine translation `_ 。 - -两种方法的区别： - -1. 两者都是对梯度的截断，但截断时机不同，前者在 :code:`optimzier` 更新网络参数时应用；后者在激活函数反向计算时被调用； -2. 截断对象不同：前者截断可学习参数的梯度，后者截断回传给前层的梯度; - -除此之外，还可以通过减小学习律或者对数据进行归一化处理来解决这类问题。 - -15. 编译安装后执行 import paddle.v2 as paddle 报ImportError: No module named v2 ------------------------------------------------------------------------------------------- -先查看一下是否曾经安装过paddle v1版本，有的话需要先卸载： - -pip uninstall py_paddle paddle - -然后安装paddle的python环境, 在build目录下执行 - -pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl - -16. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 ---------------------------------------------------------------------- - -PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，请直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 - -将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时可以跳过PaddlePaddle模型参数文件的头信息。若在PaddlePaddle编译时，未指定按照double精度编译，默认情况下按照float精度计算，保存的参数也是float类型。这时在使用 :code:`numpy.array` 时，一般设置 :code:`dtype=float32` 。示例如下： - -.. code-block:: python - - def read_parameter(fname, width): - s = open(fname).read() - # skip header - vec = np.fromstring(s[16:], dtype=np.float32) - # width is the size of the corresponding layer - np.savetxt(fname + ".csv", vec.reshape(width, -1), - fmt="%.6f", delimiter=",") - - -将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面的代码将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 - -.. code-block:: python - - def gen_rand_param(param_file, width, height, need_trans): - np.random.seed() - header = struct.pack("iil", 0, 4, height * width) - param = np.float32(np.random.rand(height, width)) - with open(param_file, "w") as fparam: - fparam.write(header + param.tostring()) - -17. 如何加载预训练参数 ------------------------------- - -* 对加载预训练参数的层，设置其参数属性 :code:`is_static=True`，使该层的参数在训练过程中保持不变。以embedding层为例，代码如下： - -.. code-block:: python - - emb_para = paddle.attr.Param(name='emb', is_static=True) - paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) - - -* 从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。以embedding层为例，代码如下： - -.. code-block:: python - - def load_parameter(file_name, h, w): - with open(file_name, 'rb') as f: - f.read(16) # skip header. - return np.fromfile(f, dtype=np.float32).reshape(h, w) - - parameters = paddle.parameters.create(my_cost) - parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) - -18. 集群多节点训练，日志中保存均为网络通信类错误 ------------------------------------------------------------ - -集群多节点训练，日志报错为网络通信类错误，比如 :code:`Connection reset by peer` 等。 -此类报错通常是由于某一个节点的错误导致这个节点的训练进程退出，从而引发其他节点无法连接导致，可以参考下面的步骤排查： - -* 从 :code:`train.log` ， :code:`server.log` 找到最早报错的地方，查看是否是其他错误引发的报错（比如FPE，内存不足，磁盘空间不足等）。 - -* 如果发现最早的报错就是网络通信的问题，很有可能是非独占方式执行导致的端口冲突，可以联系OP，看当前MPI集群是否支持resource=full参数提交，如果支持增加此参数提交，并更换job 端口。 - -* 如果当前MPI集群并不支持任务独占模式，可以联系OP是否可以更换集群或升级当前集群。 - -19. 如何调用 infer 接口输出多个layer的预测结果 ------------------------------------------------------------ - -* 将需要输出的层作为 :code:`paddle.inference.Inference()` 接口的 :code:`output_layer` 参数输入，代码如下： - -.. code-block:: python - - inferer = paddle.inference.Inference(output_layer=[layer1, layer2], parameters=parameters) - -* 指定要输出的字段进行输出。以输出 :code:`value` 字段为例，代码如下： - -.. code-block:: python - - out = inferer.infer(input=data_batch, field=["value"]) - -需要注意的是： - -* 如果指定了2个layer作为输出层，实际上需要的输出结果是两个矩阵； -* 假设第一个layer的输出A是一个 N1 * M1 的矩阵，第二个 Layer 的输出B是一个 N2 * M2 的矩阵； -* paddle.v2 默认会将A和B 横向拼接，当N1 和 N2 大小不一样时，会报如下的错误： - -.. code-block:: python - - ValueError: all the input array dimensions except for the concatenation axis must match exactly - -多个层的输出矩阵的高度不一致导致拼接失败，这种情况常常发生在： - -* 同时输出序列层和非序列层； -* 多个输出层处理多个不同长度的序列; - -此时可以在调用infer接口时通过设置 :code:`flatten_result=False` , 跳过“拼接”步骤，来解决上面的问题。这时，infer接口的返回值是一个python list: - -* list 中元素的个数等于网络中输出层的个数； -* list 中每个元素是一个layer的输出结果矩阵，类型是numpy的ndarray； -* 每一个layer输出矩阵的高度，在非序列输入时：等于样本数；序列输入时等于：输入序列中元素的总数；宽度等于配置中layer的size； - -20. :code:`paddle.layer.memory` 的参数 :code:`name` 如何使用 -------------------------------------------------------------- - -* :code:`paddle.layer.memory` 用于获取特定layer上一时间步的输出，该layer是通过参数 :code:`name` 指定，即，:code:`paddle.layer.memory` 会关联参数 :code:`name` 取值相同的layer，并将该layer上一时间步的输出作为自身当前时间步的输出。 - -* PaddlePaddle的所有layer都有唯一的name，用户通过参数 :code:`name` 设定，当用户没有显式设定时，PaddlePaddle会自动设定。而 :code:`paddle.layer.memory` 不是真正的layer，其name由参数 :code:`memory_name` 设定，当用户没有显式设定时，PaddlePaddle会自动设定。:code:`paddle.layer.memory` 的参数 :code:`name` 用于指定其要关联的layer，需要用户显式设定。 - -21. 两种使用 drop_out 的方法有何区别？ ------------------------------------------------------ - -* 在PaddlePaddle中使用dropout有两种方式 - - * 在相应layer的 :code:`layer_atter` 设置 :code:`drop_rate`，以 :code:`paddle.layer.fc` 为例，代码如下： - - .. code-block:: python - - fc = paddle.layer.fc(input=input, layer_attr=paddle.attr.ExtraLayerAttribute(drop_rate=0.5)) - - * 使用 :code:`paddle.layer.dropout`，以 :code:`paddle.layer.fc` 为例，代码如下： - - .. code-block:: python - - fc = paddle.layer.fc(input=input) - drop_fc = paddle.layer.dropout(input=fc, dropout_rate=0.5) - -* :code:`paddle.layer.dropout` 实际上使用了 :code:`paddle.layer.add_to`，并在该layer里采用第一种方式设置 :code:`drop_rate` 来使用dropout的。这种方式对内存消耗较大。 - -* PaddlePaddle在激活函数里实现dropout，而不是在layer里实现。 - -* :code:`paddle.layer.lstmemory`、:code:`paddle.layer.grumemory`、:code:`paddle.layer.recurrent` 不是通过一般的方式来实现对输出的激活，所以不能采用第一种方式在这几个layer里设置 :code:`drop_rate` 来使用dropout。若要对这几个layer使用dropout，可采用第二种方式，即使用 :code:`paddle.layer.dropout`。 - -22. 如何设置学习率退火（learning rate annealing） ------------------------------------------------- - -在相应的优化算法里设置learning_rate_schedule及相关参数，以使用Adam算法为例，代码如下： - -.. code-block:: python - - optimizer = paddle.optimizer.Adam( - learning_rate=1e-3, - learning_rate_decay_a=0.5, - learning_rate_decay_b=0.75, - learning_rate_schedule="poly",) - -PaddlePaddle目前支持8种learning_rate_schedule，这8种learning_rate_schedule及其对应学习率计算方式如下： - -* "constant" - - lr = learning_rate - -* "poly" - - lr = learning_rate * pow(1 + learning_rate_decay_a * num_samples_processed, -learning_rate_decay_b) - - 其中，num_samples_processed为已训练样本数，下同。 - -* "caffe_poly" - - lr = learning_rate * pow(1.0 - num_samples_processed / learning_rate_decay_a, learning_rate_decay_b) - -* "exp" - - lr = learning_rate * pow(learning_rate_decay_a, num_samples_processed / learning_rate_decay_b) - -* "discexp" - - lr = learning_rate * pow(learning_rate_decay_a, floor(num_samples_processed / learning_rate_decay_b)) - -* "linear" - - lr = max(learning_rate - learning_rate_decay_a * num_samples_processed, learning_rate_decay_b) - -* "manual" - - 这是一种按已训练样本数分段取值的学习率退火方法。使用该learning_rate_schedule时，用户通过参数 :code:`learning_rate_args` 设置学习率衰减因子分段函数，当前的学习率为所设置 :code:`learning_rate` 与当前的衰减因子的乘积。以使用Adam算法为例，代码如下： - - .. code-block:: python - - optimizer = paddle.optimizer.Adam( - learning_rate=1e-3, - learning_rate_schedule="manual", - learning_rate_args="1000:1.0,2000:0.9,3000:0.8",) - - 在该示例中，当已训练样本数小于等于1000时，学习率为 :code:`1e-3 * 1.0`；当已训练样本数大于1000小于等于2000时，学习率为 :code:`1e-3 * 0.9`；当已训练样本数大于2000时，学习率为 :code:`1e-3 * 0.8`。 - -* "pass_manual" - - 这是一种按已训练pass数分段取值的学习率退火方法。使用该learning_rate_schedule时，用户通过参数 :code:`learning_rate_args` 设置学习率衰减因子分段函数，当前的学习率为所设置 :code:`learning_rate` 与当前的衰减因子的乘积。以使用Adam算法为例，代码如下： - - .. code-block:: python - - optimizer = paddle.optimizer.Adam( - learning_rate=1e-3, - learning_rate_schedule="manual", - learning_rate_args="1:1.0,2:0.9,3:0.8",) - - 在该示例中，当已训练pass数小于等于1时，学习率为 :code:`1e-3 * 1.0`；当已训练pass数大于1小于等于2时，学习率为 :code:`1e-3 * 0.9`；当已训练pass数大于2时，学习率为 :code:`1e-3 * 0.8`。 - -23. 出现 :code:`Duplicated layer name` 错误怎么办 --------------------------------------------------- - -出现该错误的原因一般是用户对不同layer的参数 :code:`name` 设置了相同的取值。遇到该错误时，先找出参数 :code:`name` 取值相同的layer，然后将这些layer的参数 :code:`name` 设置为不同的值。 - -24. PaddlePaddle 中不同的 recurrent layer 的区别 --------------------------------------------------- -以LSTM为例，在PaddlePaddle中包含以下 recurrent layer： - -* :code:`paddle.layer.lstmemory` -* :code:`paddle.networks.simple_lstm` -* :code:`paddle.networks.lstmemory_group` -* :code:`paddle.networks.bidirectional_lstm` - -按照具体实现方式可以归纳为2类： - -1. 由 recurrent_group 实现的 recurrent layer： - - * 用户在使用这一类recurrent layer时，可以访问由recurrent unit在一个时间步内计算得到的中间值（例如：hidden states, memory cells等）； - * 上述的 :code:`paddle.networks.lstmemory_group` 是这一类的 recurrent layer ； - -2. 将recurrent layer作为一个整体来实现： - - * 用户在使用这一类recurrent layer，只能访问它们的输出值； - * 上述的 :code:`paddle.networks.lstmemory_group` 、 :code:`paddle.networks.simple_lstm` 和 :code:`paddle.networks.bidirectional_lstm` 属于这一类的实现； - -将recurrent layer作为一个整体来实现， 能够针对CPU和GPU的计算做更多优化， 所以相比于recurrent group的实现方式， 第二类 recurrent layer 计算效率更高。 在实际应用中，如果用户不需要访问LSTM的中间变量，而只需要获得recurrent layer计算的输出，我们建议使用第二类实现。 - -此外，关于LSTM, PaddlePaddle中还包含 :code:`paddle.networks.lstmemory_unit` 这一计算单元： - - * 不同于上述介绍的recurrent layer , :code:`paddle.networks.lstmemory_unit` 定义了LSTM单元在一个时间步内的计算过程，它并不是一个完整的recurrent layer，也不能接收序列数据作为输入； - * :code:`paddle.networks.lstmemory_unit` 只能在recurrent_group中作为step function使用； + build_and_install/index_cn.rst + model/index_cn.rst + parameter/index_cn.rst + local/index_cn.rst + cluster/index_cn.rst diff --git a/doc/faq/local/index_cn.rst b/doc/faq/local/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..75c4ba028e497e29e9030a86514348726d9c0a80 --- /dev/null +++ b/doc/faq/local/index_cn.rst @@ -0,0 +1,213 @@ +############### +本地训练与预测 +############### + +.. contents:: + +1. 如何减少内存占用 +------------------- + +神经网络的训练本身是一个非常消耗内存和显存的工作，经常会消耗数10GB的内存和数GB的显存。 +PaddlePaddle的内存占用主要分为如下几个方面\: + +* DataProvider缓冲池内存（只针对内存） +* 神经元激活内存（针对内存和显存） +* 参数内存 （针对内存和显存） +* 其他内存杂项 + +其中，其他内存杂项是指PaddlePaddle本身所用的一些内存，包括字符串分配，临时变量等等，暂不考虑在内。 + +减少DataProvider缓冲池内存 +++++++++++++++++++++++++++ + +PyDataProvider使用的是异步加载，同时在内存里直接随即选取数据来做Shuffle。即 + +.. graphviz:: + + digraph { + rankdir=LR; + 数据文件 -> 内存池 -> PaddlePaddle训练 + } + +所以，减小这个内存池即可减小内存占用，同时也可以加速开始训练前数据载入的过程。但是，这 +个内存池实际上决定了shuffle的粒度。所以，如果将这个内存池减小，又要保证数据是随机的， +那么最好将数据文件在每次读取之前做一次shuffle。可能的代码为 + +.. literalinclude:: src/reduce_min_pool_size.py + +这样做可以极大的减少内存占用，并且可能会加速训练过程，详细文档参考 :ref:`api_pydataprovider2` 。 + +神经元激活内存 +++++++++++++++ + +神经网络在训练的时候，会对每一个激活暂存一些数据，如神经元激活值等。 +在反向传递的时候，这些数据会被用来更新参数。这些数据使用的内存主要和两个参数有关系， +一是batch size，另一个是每条序列(Sequence)长度。所以，其实也是和每个mini-batch中包含 +的时间步信息成正比。 + +所以做法可以有两种： + +* 减小batch size。 即在网络配置中 :code:`settings(batch_size=1000)` 设置成一个小一些的值。但是batch size本身是神经网络的超参数，减小batch size可能会对训练结果产生影响。 +* 减小序列的长度，或者直接扔掉非常长的序列。比如，一个数据集大部分序列长度是100-200, + 但是突然有一个10000长的序列，就很容易导致内存超限，特别是在LSTM等RNN中。 + +参数内存 +++++++++ + +PaddlePaddle支持非常多的优化算法(Optimizer)，不同的优化算法需要使用不同大小的内存。 +例如使用 :code:`adadelta` 算法，则需要使用等于权重参数规模大约5倍的内存。举例，如果参数保存下来的模型目录 +文件为 :code:`100M`， 那么该优化算法至少需要 :code:`500M` 的内存。 + +可以考虑使用一些优化算法，例如 :code:`momentum`。 + +2. 如何加速训练速度 +------------------- + +加速PaddlePaddle训练可以考虑从以下几个方面\： + +* 减少数据载入的耗时 +* 加速训练速度 +* 利用分布式训练驾驭更多的计算资源 + +减少数据载入的耗时 +++++++++++++++++++ + +使用\ :code:`pydataprovider`\ 时，可以减少缓存池的大小，同时设置内存缓存功能，即可以极大的加速数据载入流程。 +:code:`DataProvider` 缓存池的减小，和之前减小通过减小缓存池来减小内存占用的原理一致。 + +.. literalinclude:: src/reduce_min_pool_size.py + +同时 :code:`@provider` 接口有一个 :code:`cache` 参数来控制缓存方法，将其设置成 :code:`CacheType.CACHE_PASS_IN_MEM` 的话，会将第一个 :code:`pass` (过完所有训练数据即为一个pass)生成的数据缓存在内存里，在之后的 :code:`pass` 中，不会再从 :code:`python` 端读取数据，而是直接从内存的缓存里读取数据。这也会极大减少数据读入的耗时。 + + +加速训练速度 +++++++++++++ + +PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`sparse_binary_vector` 、 :code:`sparse_vector` 、或者 :code:`integer_value` 的任一一种。同时，与这个训练数据交互的Layer，需要将其Parameter设置成 sparse 更新模式，即设置 :code:`sparse_update=True` + +这里使用简单的 :code:`word2vec` 训练语言模型距离，具体使用方法为\: + +使用一个词前两个词和后两个词，来预测这个中间的词。这个任务的DataProvider为\: + +.. literalinclude:: src/word2vec_dataprovider.py + +这个任务的配置为\: + +.. literalinclude:: src/word2vec_config.py + + +利用更多的计算资源 +++++++++++++++++++ + +利用更多的计算资源可以分为一下几个方式来进行\: + +* 单机CPU训练 + + * 使用多线程训练。设置命令行参数 :code:`trainer_count`。 + +* 单机GPU训练 + + * 使用显卡训练。设置命令行参数 :code:`use_gpu`。 + * 使用多块显卡训练。设置命令行参数 :code:`use_gpu` 和 :code:`trainer_count` 。 + +* 多机训练 + + * 请参考 :ref:`cluster_train` 。 + +3. 如何指定GPU设备 +------------------ + +例如机器上有4块GPU，编号从0开始，指定使用2、3号GPU： + +* 方式1：通过 `CUDA_VISIBLE_DEVICES `_ 环境变量来指定特定的GPU。 + +.. code-block:: bash + + env CUDA_VISIBLE_DEVICES=2,3 paddle train --use_gpu=true --trainer_count=2 + +* 方式2：通过命令行参数 ``--gpu_id`` 指定。 + +.. code-block:: bash + + paddle train --use_gpu=true --trainer_count=2 --gpu_id=2 + + +4. 训练过程中出现 :code:`Floating point exception`, 训练因此退出怎么办? +------------------------------------------------------------------------ + +Paddle二进制在运行时捕获了浮点数异常，只要出现浮点数异常(即训练过程中出现NaN或者Inf)，立刻退出。浮点异常通常的原因是浮点数溢出、除零等问题。 +主要原因包括两个方面: + +* 训练过程中参数或者训练过程中的梯度尺度过大，导致参数累加，乘除等时候，导致了浮点数溢出。 +* 模型一直不收敛，发散到了一个数值特别大的地方。 +* 训练数据有问题，导致参数收敛到了一些奇异的情况。或者输入数据尺度过大，有些特征的取值达到数百万，这时进行矩阵乘法运算就可能导致浮点数溢出。 + +这里有两种有效的解决方法： + +1. 设置 :code:`gradient_clipping_threshold` 参数，示例代码如下： + +.. code-block:: python + +optimizer = paddle.optimizer.RMSProp( + learning_rate=1e-3, + gradient_clipping_threshold=10.0, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + +具体可以参考 `nmt_without_attention `_ 示例。 + +2. 设置 :code:`error_clipping_threshold` 参数，示例代码如下： + +.. code-block:: python + +decoder_inputs = paddle.layer.fc( + act=paddle.activation.Linear(), + size=decoder_size * 3, + bias_attr=False, + input=[context, current_word], + layer_attr=paddle.attr.ExtraLayerAttribute( + error_clipping_threshold=100.0)) + +完整代码可以参考示例 `machine translation `_ 。 + +两种方法的区别： + +1. 两者都是对梯度的截断，但截断时机不同，前者在 :code:`optimzier` 更新网络参数时应用；后者在激活函数反向计算时被调用； +2. 截断对象不同：前者截断可学习参数的梯度，后者截断回传给前层的梯度; + +除此之外，还可以通过减小学习律或者对数据进行归一化处理来解决这类问题。 + +5. 如何调用 infer 接口输出多个layer的预测结果 +----------------------------------------------- + +* 将需要输出的层作为 :code:`paddle.inference.Inference()` 接口的 :code:`output_layer` 参数输入，代码如下： + +.. code-block:: python + + inferer = paddle.inference.Inference(output_layer=[layer1, layer2], parameters=parameters) + +* 指定要输出的字段进行输出。以输出 :code:`value` 字段为例，代码如下： + +.. code-block:: python + + out = inferer.infer(input=data_batch, field=["value"]) + +需要注意的是： + +* 如果指定了2个layer作为输出层，实际上需要的输出结果是两个矩阵； +* 假设第一个layer的输出A是一个 N1 * M1 的矩阵，第二个 Layer 的输出B是一个 N2 * M2 的矩阵； +* paddle.v2 默认会将A和B 横向拼接，当N1 和 N2 大小不一样时，会报如下的错误： + +.. code-block:: python + + ValueError: all the input array dimensions except for the concatenation axis must match exactly + +多个层的输出矩阵的高度不一致导致拼接失败，这种情况常常发生在： + +* 同时输出序列层和非序列层； +* 多个输出层处理多个不同长度的序列; + +此时可以在调用infer接口时通过设置 :code:`flatten_result=False` , 跳过“拼接”步骤，来解决上面的问题。这时，infer接口的返回值是一个python list: + +* list 中元素的个数等于网络中输出层的个数； +* list 中每个元素是一个layer的输出结果矩阵，类型是numpy的ndarray； +* 每一个layer输出矩阵的高度，在非序列输入时：等于样本数；序列输入时等于：输入序列中元素的总数；宽度等于配置中layer的size； diff --git a/doc/faq/src/reduce_min_pool_size.py b/doc/faq/local/src/reduce_min_pool_size.py similarity index 100% rename from doc/faq/src/reduce_min_pool_size.py rename to doc/faq/local/src/reduce_min_pool_size.py diff --git a/doc/faq/src/word2vec_config.py b/doc/faq/local/src/word2vec_config.py similarity index 100% rename from doc/faq/src/word2vec_config.py rename to doc/faq/local/src/word2vec_config.py diff --git a/doc/faq/src/word2vec_dataprovider.py b/doc/faq/local/src/word2vec_dataprovider.py similarity index 100% rename from doc/faq/src/word2vec_dataprovider.py rename to doc/faq/local/src/word2vec_dataprovider.py diff --git a/doc/faq/model/index_cn.rst b/doc/faq/model/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..b47bbe05bdb39d1ade9434a7e54bf6ca88a91cc9 --- /dev/null +++ b/doc/faq/model/index_cn.rst @@ -0,0 +1,69 @@ +######### +模型配置 +######### + +.. contents:: + +1. 出现 :code:`Duplicated layer name` 错误怎么办 +-------------------------------------------------- + +出现该错误的原因一般是用户对不同layer的参数 :code:`name` 设置了相同的取值。遇到该错误时，先找出参数 :code:`name` 取值相同的layer，然后将这些layer的参数 :code:`name` 设置为不同的值。 + +2. :code:`paddle.layer.memory` 的参数 :code:`name` 如何使用 +------------------------------------------------------------- + +* :code:`paddle.layer.memory` 用于获取特定layer上一时间步的输出，该layer是通过参数 :code:`name` 指定，即，:code:`paddle.layer.memory` 会关联参数 :code:`name` 取值相同的layer，并将该layer上一时间步的输出作为自身当前时间步的输出。 + +* PaddlePaddle的所有layer都有唯一的name，用户通过参数 :code:`name` 设定，当用户没有显式设定时，PaddlePaddle会自动设定。而 :code:`paddle.layer.memory` 不是真正的layer，其name由参数 :code:`memory_name` 设定，当用户没有显式设定时，PaddlePaddle会自动设定。:code:`paddle.layer.memory` 的参数 :code:`name` 用于指定其要关联的layer，需要用户显式设定。 + +3. 两种使用 drop_out 的方法有何区别 +------------------------------------ + +* 在PaddlePaddle中使用dropout有两种方式 + + * 在相应layer的 :code:`layer_atter` 设置 :code:`drop_rate`，以 :code:`paddle.layer.fc` 为例，代码如下： + + .. code-block:: python + + fc = paddle.layer.fc(input=input, layer_attr=paddle.attr.ExtraLayerAttribute(drop_rate=0.5)) + + * 使用 :code:`paddle.layer.dropout`，以 :code:`paddle.layer.fc` 为例，代码如下： + + .. code-block:: python + + fc = paddle.layer.fc(input=input) + drop_fc = paddle.layer.dropout(input=fc, dropout_rate=0.5) + +* :code:`paddle.layer.dropout` 实际上使用了 :code:`paddle.layer.add_to`，并在该layer里采用第一种方式设置 :code:`drop_rate` 来使用dropout的。这种方式对内存消耗较大。 + +* PaddlePaddle在激活函数里实现dropout，而不是在layer里实现。 + +* :code:`paddle.layer.lstmemory`、:code:`paddle.layer.grumemory`、:code:`paddle.layer.recurrent` 不是通过一般的方式来实现对输出的激活，所以不能采用第一种方式在这几个layer里设置 :code:`drop_rate` 来使用dropout。若要对这几个layer使用dropout，可采用第二种方式，即使用 :code:`paddle.layer.dropout`。 + +4. 不同的 recurrent layer 的区别 +---------------------------------- +以LSTM为例，在PaddlePaddle中包含以下 recurrent layer： + +* :code:`paddle.layer.lstmemory` +* :code:`paddle.networks.simple_lstm` +* :code:`paddle.networks.lstmemory_group` +* :code:`paddle.networks.bidirectional_lstm` + +按照具体实现方式可以归纳为2类： + +1. 由 recurrent_group 实现的 recurrent layer： + + * 用户在使用这一类recurrent layer时，可以访问由recurrent unit在一个时间步内计算得到的中间值（例如：hidden states, memory cells等）； + * 上述的 :code:`paddle.networks.lstmemory_group` 是这一类的 recurrent layer ； + +2. 将recurrent layer作为一个整体来实现： + + * 用户在使用这一类recurrent layer，只能访问它们的输出值； + * 上述的 :code:`paddle.networks.lstmemory_group` 、 :code:`paddle.networks.simple_lstm` 和 :code:`paddle.networks.bidirectional_lstm` 属于这一类的实现； + +将recurrent layer作为一个整体来实现， 能够针对CPU和GPU的计算做更多优化， 所以相比于recurrent group的实现方式， 第二类 recurrent layer 计算效率更高。 在实际应用中，如果用户不需要访问LSTM的中间变量，而只需要获得recurrent layer计算的输出，我们建议使用第二类实现。 + +此外，关于LSTM, PaddlePaddle中还包含 :code:`paddle.networks.lstmemory_unit` 这一计算单元： + + * 不同于上述介绍的recurrent layer , :code:`paddle.networks.lstmemory_unit` 定义了LSTM单元在一个时间步内的计算过程，它并不是一个完整的recurrent layer，也不能接收序列数据作为输入； + * :code:`paddle.networks.lstmemory_unit` 只能在recurrent_group中作为step function使用； diff --git a/doc/faq/parameter/index_cn.rst b/doc/faq/parameter/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..c721b623183cc7d8d17e2c9fb1635ea07b8970cc --- /dev/null +++ b/doc/faq/parameter/index_cn.rst @@ -0,0 +1,201 @@ +######### +参数设置 +######### + +.. contents:: + +1. 如何选择SGD算法的学习率 +-------------------------- + +在采用sgd/async_sgd进行训练时，一个重要的问题是选择正确的learning_rate。如果learning_rate太大，那么训练有可能不收敛，如果learning_rate太小，那么收敛可能很慢，导致训练时间过长。 + +通常做法是从一个比较大的learning_rate开始试，如果不收敛，那减少学习率10倍继续试验，直到训练收敛为止。那么如何判断训练不收敛呢？可以估计出如果模型采用不变的输出最小的cost0是多少。 + +如果训练过程的的cost明显高于这个常数输出的cost，那么我们可以判断为训练不收敛。举一个例子，假如我们是三分类问题，采用multi-class-cross-entropy作为cost，数据中0,1,2三类的比例为 :code:`0.2, 0.5, 0.3` , 那么常数输出所能达到的最小cost是 :code:`-(0.2*log(0.2)+0.5*log(0.5)+0.3*log(0.3))=1.03` 。如果训练一个pass（或者更早）后，cost还大于这个数，那么可以认为训练不收敛，应该降低学习率。 + +2. 如何设置学习率退火（learning rate annealing） +------------------------------------------------ + +在相应的优化算法里设置learning_rate_schedule及相关参数，以使用Adam算法为例，代码如下： + +.. code-block:: python + + optimizer = paddle.optimizer.Adam( + learning_rate=1e-3, + learning_rate_decay_a=0.5, + learning_rate_decay_b=0.75, + learning_rate_schedule="poly",) + +PaddlePaddle目前支持8种learning_rate_schedule，这8种learning_rate_schedule及其对应学习率计算方式如下： + +* "constant" + + lr = learning_rate + +* "poly" + + lr = learning_rate * pow(1 + learning_rate_decay_a * num_samples_processed, -learning_rate_decay_b) + + 其中，num_samples_processed为已训练样本数，下同。 + +* "caffe_poly" + + lr = learning_rate * pow(1.0 - num_samples_processed / learning_rate_decay_a, learning_rate_decay_b) + +* "exp" + + lr = learning_rate * pow(learning_rate_decay_a, num_samples_processed / learning_rate_decay_b) + +* "discexp" + + lr = learning_rate * pow(learning_rate_decay_a, floor(num_samples_processed / learning_rate_decay_b)) + +* "linear" + + lr = max(learning_rate - learning_rate_decay_a * num_samples_processed, learning_rate_decay_b) + +* "manual" + + 这是一种按已训练样本数分段取值的学习率退火方法。使用该learning_rate_schedule时，用户通过参数 :code:`learning_rate_args` 设置学习率衰减因子分段函数，当前的学习率为所设置 :code:`learning_rate` 与当前的衰减因子的乘积。以使用Adam算法为例，代码如下： + + .. code-block:: python + + optimizer = paddle.optimizer.Adam( + learning_rate=1e-3, + learning_rate_schedule="manual", + learning_rate_args="1000:1.0,2000:0.9,3000:0.8",) + + 在该示例中，当已训练样本数小于等于1000时，学习率为 :code:`1e-3 * 1.0`；当已训练样本数大于1000小于等于2000时，学习率为 :code:`1e-3 * 0.9`；当已训练样本数大于2000时，学习率为 :code:`1e-3 * 0.8`。 + +* "pass_manual" + + 这是一种按已训练pass数分段取值的学习率退火方法。使用该learning_rate_schedule时，用户通过参数 :code:`learning_rate_args` 设置学习率衰减因子分段函数，当前的学习率为所设置 :code:`learning_rate` 与当前的衰减因子的乘积。以使用Adam算法为例，代码如下： + + .. code-block:: python + + optimizer = paddle.optimizer.Adam( + learning_rate=1e-3, + learning_rate_schedule="manual", + learning_rate_args="1:1.0,2:0.9,3:0.8",) + + 在该示例中，当已训练pass数小于等于1时，学习率为 :code:`1e-3 * 1.0`；当已训练pass数大于1小于等于2时，学习率为 :code:`1e-3 * 0.9`；当已训练pass数大于2时，学习率为 :code:`1e-3 * 0.8`。 + +3. 如何初始化参数 +----------------- + +默认情况下，PaddlePaddle使用均值0，标准差为 :math:`\frac{1}{\sqrt{d}}` 来初始化参数。其中 :math:`d` 为参数矩阵的宽度。这种初始化方式在一般情况下不会产生很差的结果。如果用户想要自定义初始化方式，PaddlePaddle目前提供两种参数初始化的方式\: + +* 高斯分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_mean=0.0, initial_std=1.0)` +* 均匀分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0)` + +比如设置一个全连接层的参数初始化方式和bias初始化方式，可以使用如下代码。 + +.. code-block:: python + + hidden = fc_layer(input=ipt, param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0), + bias_attr=ParamAttr(initial_mean=1.0, initial_std=0.0)) + +上述代码将bias全部初始化为1.0, 同时将参数初始化为 :code:`[1.0, -1.0]` 的均匀分布。 + +4. 如何共享参数 +--------------- + +PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字的参数，会共享参数。设置参数的名字，可以使用 :code:`ParamAttr(name="YOUR_PARAM_NAME")` 来设置。更方便的设置方式，是使得要共享的参数使用同样的 :code:`ParamAttr` 对象。 + +简单的全连接网络，参数共享的配置示例为\: + +.. literalinclude:: ../../python/paddle/trainer_config_helpers/tests/configs/shared_fc.py + +这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 + +5. 如何加载预训练参数 +------------------------ + +* 对加载预训练参数的层，设置其参数属性 :code:`is_static=True`，使该层的参数在训练过程中保持不变。以embedding层为例，代码如下： + +.. code-block:: python + + emb_para = paddle.attr.Param(name='emb', is_static=True) + paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) + + +* 从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。以embedding层为例，代码如下： + +.. code-block:: python + + def load_parameter(file_name, h, w): + with open(file_name, 'rb') as f: + f.read(16) # skip header. + return np.fromfile(f, dtype=np.float32).reshape(h, w) + + parameters = paddle.parameters.create(my_cost) + parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) + +6. 存储的参数格式是什么，如何和明文进行相互转化 +-------------------------------------------------- + +PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，请直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 + +将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时可以跳过PaddlePaddle模型参数文件的头信息。若在PaddlePaddle编译时，未指定按照double精度编译，默认情况下按照float精度计算，保存的参数也是float类型。这时在使用 :code:`numpy.array` 时，一般设置 :code:`dtype=float32` 。示例如下： + +.. code-block:: python + + def read_parameter(fname, width): + s = open(fname).read() + # skip header + vec = np.fromstring(s[16:], dtype=np.float32) + # width is the size of the corresponding layer + np.savetxt(fname + ".csv", vec.reshape(width, -1), + fmt="%.6f", delimiter=",") + + +将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面的代码将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 + +.. code-block:: python + + def gen_rand_param(param_file, width, height, need_trans): + np.random.seed() + header = struct.pack("iil", 0, 4, height * width) + param = np.float32(np.random.rand(height, width)) + with open(param_file, "w") as fparam: + fparam.write(header + param.tostring()) + +7. A protocol message was rejected because it was too big +------------------------------------------------------------ + +如果在训练NLP相关模型时，出现以下错误： + +.. code-block:: bash + + [libprotobuf ERROR google/protobuf/io/coded_stream.cc:171] A protocol message was rejected because it was too big (more than 67108864 bytes). To increase the limit (or to disable these warnings), see CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h. + F1205 14:59:50.295174 14703 TrainerConfigHelper.cpp:59] Check failed: m->conf.ParseFromString(configProtoStr) + +可能的原因是：传给dataprovider的某一个args过大，一般是由于直接传递大字典导致的。错误的define_py_data_sources2类似： + +.. code-block:: python + + src_dict = dict() + for line_count, line in enumerate(open(src_dict_path, "r")): + src_dict[line.strip()] = line_count + + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={"src_dict": src_dict}) + +解决方案是：将字典的地址作为args传给dataprovider，然后在dataprovider里面根据该地址加载字典。即define_py_data_sources2应改为： + +.. code-block:: python + + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={"src_dict_path": src_dict_path}) + +完整源码可参考 `seqToseq `_ 示例。 + + diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index b7aa501db9e5c7378398fad48503f82bff893b60..bad1dbc1de9cc5bd11914fddf397857f0bda7976 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -1,14 +1,17 @@ # How to write a new operator - - [Background](#Background) - - [Implementing C++ Types](#Implementing_C++_Types) - - [Defining ProtoMaker](#Defining_ProtoMaker) - - [Defining Operator](#Defining_Operator) - - [Registering Operator](#Registering_Operator) - - [Compilation](#Compilation) - - [Python Binding](#Python_Binding) - - [Unit Tests](#Unit_Tests) - + - [Background](#background) + - [Implementing C++ Types](#implementing-c++-types) + - [Defining ProtoMaker](#defining-protoMaker) + - [Defining Operator](#defining-operator) + - [Registering Operator](#registering-operator) + - [Compilation](#compilation) + - [Python Binding](#python-binding) + - [Unit Tests](#unit-tests) + - [Testing Forward Operators](#testing-forward-operators) + - [Testing Backward Operators](#testing-backward-operators) + - [Compiling and Running](#compiling-and-running) + - [Remarks](#remarks) ## Background Here are the base types needed. For details, please refer to the design docs. @@ -179,7 +182,7 @@ Note that **different devices (CPU, GPU)share an Op definition; whether or not t `MulOp`'s CPU and GPU share the same `Kernel`. A non-sharing `OpKernel` example can be seen in [`OnehotCrossEntropyOpKernel`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/cross_entropy_op.h#L43). -To ease the writing of `OpKernel` compute, and for reusing code cross-device, `Eigen unsupported Tensor` module is used to implement `Compute` interface. To learn about how the Eigen library is used in PaddlePaddle, please see [usage document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md). +To ease the writing of `OpKernel` compute, and for reusing code cross-device, [`Eigen-unsupported Tensor`](https://bitbucket.org/eigen/eigen/src/default/unsupported/Eigen/CXX11/src/Tensor/README.md?fileviewer=file-view-default) module is used to implement `Compute` interface. To learn about how the Eigen library is used in PaddlePaddle, please see [usage document](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/use_eigen_cn.md). This concludes the forward implementation of an operator. Next its operation and kernel need to be registered in a `.cc` file. @@ -232,4 +235,122 @@ The system will automatically bind to Python and link it to a generated library. ## Unit Tests -Unit tests include comparing a forward operator's implementations on different devices, comparing a backward operator's implementation on different devices, and a scaling test for the backward operator. Here, we introduce the [unit tests for `MulOp`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py). +Unit tests for an operator include + +1. comparing a forward operator's implementations on different devices, + +2. comparing a backward operator's implementation on different devices, and + +3. a scaling test for the backward operator. + +Here, we introduce the [unit tests for `MulOp`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/framework/tests/test_mul_op.py). + +### Testing Forward Operators + +A forward operator unit test inherits `unittest.TestCase` and defines metaclass `__metaclass__ = OpTestMeta`. More concrete tests are performed in `OpTestMeta`. Testing a forward operator requires the following: + +1. Defining input, output and relevant attributes in `setUp` method. + +2. Generating random input data. + +3. Implementing the same computation logic in a Python script: + + ```python + import unittest + import numpy as np + from gradient_checker import GradientChecker, create_op + from op_test_util import OpTestMeta + + class TestMulOp(unittest.TestCase): + __metaclass__ = OpTestMeta + + def setUp(self): + self.type = "mul" + self.inputs = { + 'X': np.random.random((32, 84)).astype("float32"), + 'Y': np.random.random((84, 100)).astype("float32") + } + self.outputs = {'Out': np.dot(self.inputs['X'], self.inputs['Y'])} + ``` +Get its output, and compare it with the forward operator's own output. + +The code above first loads required packages. In addition, we have + +- `self.type = "mul" ` defines the type that is identical to what the operator's registered type. +- `self.inputs` defines input, with type `numpy.array` and initializes it. +- `self.outputs` defines output and completes the same operator computation in the Python script, and returns its result from the Python script. + +### Testing Backward Operators + +A backward operator unit test inherits `GradientChecker`, which inherits `unittest.TestCase`. As a result, **a backward operator unit test needs to be have the prefix `test_`**. + +```python +class TestMulGradOp(GradientChecker): + def setUp(self): + self.op = create_op("mul") + self.inputs = { + 'X': np.random.random((32, 84)).astype("float32"), + 'Y': np.random.random((84, 100)).astype("float32") + } + + def test_cpu_gpu_compare(self): + self.compare_grad(self.op, self.inputs) + + def test_normal(self): + # mul op will enlarge the relative error + self.check_grad( + self.op, self.inputs, ["X", "Y"], "Out", max_relative_error=0.5) + + def test_ignore_x(self): + self.check_grad( + self.op, + self.inputs, ["Y"], + "Out", + max_relative_error=0.5, + no_grad_set={"X"}) + + def test_ignore_y(self): + self.check_grad( + self.op, + self.inputs, ["X"], + "Out", + max_relative_error=0.5, + no_grad_set={"Y"}) +``` + +Some key points in the code above include: + +- `create_op("mul")` creates the backward operator's corresponding forward operator. +- `compare_grad` compares results between utilizing the CPU and the GPU. +- `test_normal` calls `check_grad` to validate scaling tests' correctness and stability through numeric methods. + - The first variable `self.op` denotes the forward operator. + - The second variable `self.inputs` denotes the input dictionary, which has its key value identical to its `ProtoMaker` definitions. + - The third variable `["X", "Y"]` appoints `X` and `Y` to be scale tested. + - The fourth variable `"Out"` points to the network's final output target `Out`. +- `test_ignore_x` and `test_ignore_y`branches test the cases where there is only one scaling input. + +### Compiling and Running + + +Any new unit testing file of the format `test_*.py` added to the director `python/paddle/v2/framework/tests` is automatically added to the project to compile. + +Note that **unlike the compile test for Ops, running unit tests requires compiling the entire project** and requires compiling with flag `WITH_TESTING` on i.e. `cmake paddle_dir -DWITH_TESTING=ON`. + +After successfully compiling the project, run the following command to run unit tests: + +```bash +make test ARGS="-R test_mul_op -V" +``` + +Or, + +```bash +ctest -R test_mul_op +``` + +## Remarks + +- Every `*_op.h` (if applicable), `*_op.cc`, and `*_op.cu` (if applicable) must be created for a unique Op. Compiling will fail if multiple operators are included per file. +- The type with which an operator is registered needs to be identical to the Op's name. Registering `REGISTER_OP(B, ...)` in `A_op.cc` will cause unit testing failures. +- If the operator does not implement a GPU kernel, please refrain from creating an empty `*_op.cu` file, or else unit tests will fail. +- If multiple operators rely on some shared methods, a file NOT named `*_op.*` can be created to store them, such as `gather.h`. diff --git a/doc/howto/dev/use_eigen_en.md b/doc/howto/dev/use_eigen_en.md new file mode 100644 index 0000000000000000000000000000000000000000..e169106e12f5d62696f1f0e7163562793b32c18c --- /dev/null +++ b/doc/howto/dev/use_eigen_en.md @@ -0,0 +1,146 @@ +## How to use Eigen in Paddle + +Essentially, a neural network is a compute graph. T data needed for the computation is stored in `Tensor`s and its computation procedure is described by `Operator`s. An `Operator` calls the `Compute` interface in its corresponding `OpKernel` and operates on the `Tensor`. + + +### Eigen Tensor Module + +The Eigen Tensor module supports powerful element-wise computation. In addition, a piece of code written using it can be run on both the CPU and the GPU. + +Note that Eigen Tensor is still being actively developed, so its tests are not completely covered and its documentation may be sparse. + +For details on Eigen Tensor module, please see [doc 1](https://github.com/RLovelett/eigen/blob/master/unsupported/Eigen/CXX11/src/Tensor/README.md) and [doc 2](https://bitbucket.org/eigen/eigen/src/default/unsupported/Eigen/CXX11/src/Tensor/README.md). + + +### paddle::framework::Tensor + +Paddle Tensor's is defined in the framework directory with the following interface: + +```cpp +class Tensor { + public: + /*! Return a pointer to mutable memory block. */ + template + inline T* data(); + + /** + * @brief Return a pointer to mutable memory block. + * @note If not exist, then allocation. + */ + template + inline T* mutable_data(platform::Place place); + + /** + * @brief Return a pointer to mutable memory block. + * + * @param[in] dims The dimensions of the memory block. + * @param[in] place The place of the memory block. + * + * @note If not exist, then allocation. + */ + template + inline T* mutable_data(DDim dims, platform::Place place); + + /*! Resize the dimensions of the memory block. */ + inline Tensor& Resize(const DDim& dims); + + /*! Return the dimensions of the memory block. */ + inline const DDim& dims() const; + + private: + /*! holds the memory block if allocated. */ + std::shared_ptr holder_; + + /*! points to dimensions of memory block. */ + DDim dim_; +}; +``` + +`Placeholder` is used to delay memory allocation; that is, we can first define a tensor, using `Resize` to configure its shape, and then call `mutuable_data` to allocate the actual memory. + +```cpp +paddle::framework::Tensor t; +paddle::platform::CPUPlace place; +// set size first +t.Resize({2, 3}); +// allocate memory on CPU later +t.mutable_data(place); +``` + +### paddle::framework::Tensor Usage +`AddOp` demonstrates Tensor's usage. + +- InferShape + +When computing a neural network's compute graph, first call every `Operator`'s `InferShape` method, and use `Resize` to configure the size of the output tensor. + +```cpp +void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_EQ(ctx.Input("X")->dims(), + ctx.Input("Y")->dims(), + "Two input of Add Op's dimension must be same."); + ctx.Output("Out")->Resize(ctx.Input("X")->dims()); +} +``` + + +- Run + +```cpp +void Compute(const framework::ExecutionContext& context) const override { + auto* input0 = context.Input("X"); + auto* input1 = context.Input("Y"); + auto* output = context.Output("Out"); + + output->mutable_data(context.GetPlace()); + + auto x = EigenVector::Flatten(*input0); + auto y = EigenVector::Flatten(*input1); + auto z = EigenVector::Flatten(*output); + + auto place = context.GetEigenDevice(); + + z.device(place) = x + y; +} +``` + + +### paddle::framework::Tensor到EigenTensor的转换 + +As shown above, in actual computation, we need to transform the input and output `Tensor`s into formats Eigen supports. We show some functions in [eigen.h](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/eigen.h) to implement the transformation from `paddle::framework::Tensor`to `EigenTensor/EigenMatrix/EigenVector/EigenScalar`. + +Using EigenTensor as an example: + +```cpp +Tensor t; +float* p = t.mutable_data(make_ddim({1, 2, 3}), platform::CPUPlace()); +for (int i = 0; i < 1 * 2 * 3; i++) { + p[i] = static_cast(i); +} + +EigenTensor::Type et = EigenTensor::From(t); +``` + +`From` is an interfacing method provided by the EigenTensor template, which implements the transformation from a `paddle::framework::Tensor` object to an EigenTensor. Since `rank` is a template parameter, it needs to be explicitly specified at the time of the transformation. + +In Eigen, tensors with different ranks are different types, with `Vector` bring a rank-1 instance. Note that `EigenVector::From` uses a transformation from an 1-dimensional Paddle tensor to a 1-dimensional Eigen tensor while `EigenVector::Flatten` reshapes a paddle tensor and flattens it into a 1-dimensional Eigen tensor. Both resulting tensors are still typed EigenVector. + +For more transformations, see the [unit tests](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/eigen_test.cc) in the `eigen_test.cc` file. + + + +### Implementing Computation + +While computing, the device interface is needed from the EigenTensors on the left hand side of the assignments. Note that the computation between EigenTensors only changes the data originally inthe Tensor and does not change all the shape information associated with the Tensor. + +```cpp +auto x = EigenVector::Flatten(*input0); +auto y = EigenVector::Flatten(*input1); +auto z = EigenVector::Flatten(*output); +auto place = context.GetEigenDevice(); +z.device(place) = x + y; +``` + +In this code segment, input0/input1/output can be Tensors of arbitrary dimension. We are calling Flatten from EigenVector, transforming a tensor of any dimension into a 1-dimensional EigenVector. After completing computation, input0/input1/output will retain the same shape information, and they can be resized using the `Resize` interface. + +Because the Eigen Tensor module is under-documented, please refer to `OpKernel`'s computation code in TensorFlow's [kernel module documentation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/kernels). diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index e535f84dba7c2726fbb70fa11ca8e9e2d29b8665..4aaa43d79612111856dd4dfc954ca2bfd8f4fa63 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -19,6 +19,7 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) proto_library(framework_proto SRCS framework.proto) cc_library(attribute SRCS attribute.cc DEPS framework_proto) +cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute) cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) @@ -26,7 +27,7 @@ cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op) diff --git a/paddle/framework/attribute.cc b/paddle/framework/attribute.cc index 510dc28c57f642786e7c64d86961c76ac80014a8..d6a2975aaa419406aef7b228e78381dbce78890d 100644 --- a/paddle/framework/attribute.cc +++ b/paddle/framework/attribute.cc @@ -24,6 +24,9 @@ static ProgramDesc* g_program_desc = nullptr; ProgramDesc& GetProgramDesc() { if (g_program_desc == nullptr) { g_program_desc = new ProgramDesc(); + auto root_block = g_program_desc->mutable_blocks()->Add(); + root_block->set_idx(0); + root_block->set_parent_idx(-1); } return *g_program_desc; } diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index 488fa38faf12ee51087643f79295f36bfd33ee22..c7559cefb6415ee141f32e4357459653564cd2ac 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -45,6 +45,21 @@ inline AttrType AttrTypeID() { Attribute GetAttrValue(const OpDesc::Attr& attr_desc); +class AttrReader { + public: + explicit AttrReader(const AttributeMap& attrs) : attrs_(attrs) {} + + template + inline const T& Get(const std::string& name) const { + PADDLE_ENFORCE(attrs_.count(name) != 0, "%s should be in AttributeMap", + name); + return boost::get(attrs_.at(name)); + } + + private: + const AttributeMap& attrs_; +}; + // check whether a value(attribute) fit a certain limit template class GreaterThanChecker { diff --git a/paddle/framework/backward.md b/paddle/framework/backward.md index 0a6d762bc8be5201ac196b4bc6107c06d07a31d7..ac60be572419b62f4beb644ff192d413c35e19bb 100644 --- a/paddle/framework/backward.md +++ b/paddle/framework/backward.md @@ -2,7 +2,7 @@ ## Motivation -In Neural Network, many model is solved by the the backpropagation algorithm(known as BP) at present. Technically it caculates the gradient of the loss function, then distributed back through the networks. Follows the chain rule, so we need a module chains the gradient operators/expressions together with to construct the backward pass. Every forward network needs a backward network to construct the full computation graph, the operator/expression's backward pass will be generated respect to forward pass. +In Neural Network, most models are solved by the backpropagation algorithm(known as **BP**) at present. Technically, BP calculates the gradient of the loss function, then propagates it back through the networks following the chain rule. Hence we need a module that chains the gradient operators/expressions together to construct the backward pass. Every forward network needs a backward network to construct the full computation graph. The operator/expression's backward pass will be generated with respect to the forward pass. ## Implementation @@ -24,9 +24,9 @@ A backward network is built up with several backward operators. Backward operato | **Operator::inputs_** | Inputs | Inputs, Outputs, OutputGradients | | **Operator::outputs_** | Outputs | InputGradients | - In most cases, there is a one-to-one correspondence between the forward and backward operators. These correspondences are recorded by a global hash map(`OpInfoMap`). To follow the philosophy of minimum core and make operators pluggable, the registry mechanism is introduced. + In most cases, there is a one-to-one relation between the forward and backward operators. These relations are recorded by a global hash map(`OpInfoMap`). To follow the philosophy of minimum core and to make operators pluggable, the registry mechanism is introduced. -For example, we have got a `mul_op`, and we can register its information and corresponding backward operator by the following macro: +For example, we have `mul_op`, and we can register its information and corresponding backward operator by the following macro: ```cpp REGISTER_OP(mul, MulOp, MulOpMaker, mul_grad, MulOpGrad); @@ -48,7 +48,7 @@ The function `BuildGradOp` will sequentially execute following processes: 1. Get the `type_` of given forward operator, and then get the corresponding backward operator's type by looking up the `OpInfoMap`. -2. Build two maps named `inputs` and `outputs` to temporary storage backward operator's inputs and outputs. Copy forward operator's `inputs_` and `outputs_` to map `inputs`, except these, are not necessary for gradient computing. +2. Build two maps named `inputs` and `outputs` to temporarily store backward operator's inputs and outputs. Copy forward operator's `inputs_` and `outputs_` to map `inputs`, except these, are not necessary for gradient computing. 3. Add forward inputs' gradient variables into map `output`, adding forward outputs' gradient variables into map `input`. @@ -56,11 +56,11 @@ The function `BuildGradOp` will sequentially execute following processes: ### Backward Network Building -A backward network is a series of backward operators. The main idea of building a backward network is creating backward operators in the inverted sequence and append them together one by one. There is some corner case need to process specially. +A backward network is a series of backward operators. The main idea of building a backward network is creating backward operators in the inverted sequence and appending them together one by one. There are some corner cases that need special processing. 1. Op - When the input forward network is an Op, return its gradient Operator Immediately. If all of its outputs are in no gradient set, then return a special `NOP`. + When the input forward network is an Op, return its gradient Operator immediately. If all of its outputs are in no gradient set, then return a special `NOP`. 2. NetOp @@ -68,33 +68,33 @@ A backward network is a series of backward operators. The main idea of building 3. RnnOp - RnnOp is a nested stepnet operator. Backward module need to recusively call `Backward` for every stepnet. + RnnOp is a nested stepnet operator. Backward module needs to recusively call `Backward` for every stepnet. 4. Sharing Variables - **sharing variables**. As illustrated in the pictures, two operator's share the same variable name of W@GRAD, which will overwrite their sharing input variable. + As illustrated in the figure 1 and figure 2, two operators share the same variable name **W@GRAD**, which will overwrite their shared input variable.


-​ pic 1. Sharing variables in operators. +​ Figure 1. Sharing variables in operators.

-​ Sharing variable between operators or same input variable used in multiple operators leads to a duplicate gradient variable. As demo show above, we need to rename gradient name recursively and add a generic add operator to replace the overwrite links. +​ Sharing variable between operators or same input variable used in multiple operators can lead to duplicate gradient variables. As illustrated in figure 2, we need to rename the gradient names recursively and add a generic add operator to prevent overwriting.


-​ pic 2. Replace sharing variable's gradient with `Add` operator. +​ Figure 2. Replace sharing variable's gradient with `Add` operator.

-​ Because our framework finds variables accord to their names, we need to rename the output links. We add a suffix of number to represent its position in clockwise. +​ Because the framework finds variables according to their names, we need to rename the output links. We add an integer suffix to represent its position in the clockwise direction. -5. Part of Gradient is Zero. +5. Part of the Gradient is Zero. - In the whole graph, there is some case of that one operator's gradient is not needed, but its input's gradient is a dependency link of other operator, we need to fill a same shape gradient matrix in the position. In our implement, we insert a special `fillZeroLike` operator. + In the whole graph, there is some case of that one operator's gradient is not needed, but its input's gradient is a dependency link of other operator, we need to fill a same shape gradient matrix in the position. In our implementation, we insert a special `fillZeroLike` operator. Follow these rules above, then collect the sub graph `OutputGradients`/`InputGradients` as the NetOp's and return it. diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc new file mode 100644 index 0000000000000000000000000000000000000000..9570aedfdda332b797a8f348e0f6cf81bb2aee2f --- /dev/null +++ b/paddle/framework/block_desc.cc @@ -0,0 +1,89 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/block_desc.h" +#include "paddle/framework/program_desc.h" + +namespace paddle { +namespace framework { + +VarDescBind *BlockDescBind::NewVar(const std::string &name) { + need_update_ = true; + auto it = vars_.find(name); + PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name); + auto var = new VarDescBind(name); + vars_[name].reset(var); + return var; +} + +VarDescBind *BlockDescBind::Var(const std::string &name) const { + auto it = vars_.find(name); + PADDLE_ENFORCE(it != vars_.end(), + "Can not find variable %s in current block.", name); + return it->second.get(); +} + +std::vector BlockDescBind::AllVars() const { + std::vector res; + for (const auto &p : vars_) { + res.push_back(p.second.get()); + } + return res; +} + +OpDescBind *BlockDescBind::AppendOp() { + need_update_ = true; + ops_.emplace_back(new OpDescBind()); + return ops_.back().get(); +} + +OpDescBind *BlockDescBind::PrependOp() { + need_update_ = true; + ops_.emplace_front(new OpDescBind()); + return ops_.front().get(); +} + +std::vector BlockDescBind::AllOps() const { + std::vector res; + for (const auto &op : ops_) { + res.push_back(op.get()); + } + return res; +} + +void BlockDescBind::Sync() { + if (need_update_) { + auto &op_field = *this->desc_->mutable_ops(); + op_field.Clear(); + op_field.Reserve(static_cast(ops_.size())); + for (auto &op_desc : ops_) { + op_field.AddAllocated(op_desc->Proto()); + } + need_update_ = false; + } +} + +BlockDescBind *BlockDescBind::ParentBlock() const { + if (this->desc_->parent_idx() == -1) { + return nullptr; + } + return prog_->Block(static_cast(this->desc_->parent_idx())); +} + +void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) { + BlockDesc *desc = block.RawPtr(); + this->attrs_[name] = desc; +} +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h new file mode 100644 index 0000000000000000000000000000000000000000..1a1135bab44cd27bb7d784c3b486188aa40635e4 --- /dev/null +++ b/paddle/framework/block_desc.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include "paddle/framework/op_desc.h" +#include "paddle/framework/var_desc.h" + +namespace paddle { +namespace framework { + +class ProgramDescBind; + +// Each Protobuf Message, we provide a XXXBind class. In that class, we optimize +// read/write speed. Only when we want the protobuf message, the local changes +// will be synchronized (by `Sync` method). + +class BlockDescBind { + public: + BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) + : prog_(prog), desc_(desc), need_update_(false) {} + + BlockDescBind(const BlockDescBind &o) = delete; + BlockDescBind &operator=(const BlockDescBind &o) = delete; + + int32_t ID() const { return desc_->idx(); } + + int32_t Parent() const { return desc_->parent_idx(); } + + VarDescBind *NewVar(const std::string &name_bytes); + + VarDescBind *Var(const std::string &name_bytes) const; + + std::vector AllVars() const; + + BlockDescBind *ParentBlock() const; + + OpDescBind *AppendOp(); + + OpDescBind *PrependOp(); + + std::vector AllOps() const; + + void Sync(); + + BlockDesc *RawPtr() { return desc_; } + + private: + ProgramDescBind *prog_; // not_own + BlockDesc *desc_; // not_own + bool need_update_; + + std::deque> ops_; + std::unordered_map> vars_; +}; +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 3c349637cdbe59b2cf9a1ea28e7715f4181f9293..5b7badf89c1714331bae9fc8cf94c8da2c66dbad 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -72,6 +72,22 @@ bool operator==(const LoD& a, const LoD& b) { return true; } +size_t LoDTensor::NumElements(size_t level, size_t idx) const { + PADDLE_ENFORCE_LT(level, NumLevels()); + PADDLE_ENFORCE_LT(idx, NumElements(level)); + // the last level of LoD, just return number of records in Tensor + if (level == NumLevels() - 1) { + return lod_[level][idx + 1] - lod_[level][idx]; + } + // high level of LoD, and there is another lower level, return number of + // lower-level elements + auto tmp = SliceInLevel(lod_, level, idx, idx + 1); + PADDLE_ENFORCE_GE(tmp.size(), 2); + // there is a 0 as a placeholder stored in LoD, so the number of elements + // equals lod.size() - 1 + return tmp[1].size() - 1; +} + void LoDTensor::ShrinkLevels(size_t level_begin, size_t level_end) { auto new_lod = framework::SliceLevels(lod_, level_begin, level_end); lod_ = new_lod; diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index 82f58464264c6871b51251e0feae3d5ca076cd2b..49786a4a6635f1b39356dbf9633c4e7da443f04e 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -38,6 +38,18 @@ using Vector = thrust::host_vector< T, thrust::system::cuda::experimental::pinned_allocator>; #endif +/* + * 3-level LoD stores + * + * 0 10 20 + * 0 5 10 15 20 + * 0 2 5 7 10 12 15 20 + * + * - in a level, each element indicates offset in the underlying Tensor + * - the first element should be 0 and that indicates that this sequence start + * from 0 + * - each sequence's begin and end(no-inclusive) is level[id, id+1] + */ using LoD = std::vector>; LoD SliceLevels(const LoD& in, size_t level_begin, size_t level_end); @@ -65,11 +77,8 @@ class LoDTensor : public Tensor { * Get a element from LoD. */ size_t lod_element(size_t level, size_t elem) const { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); - PADDLE_ENFORCE(elem < NumElements(level), - "element begin [%d] out of range [%d]", elem, - NumElements(level)); + PADDLE_ENFORCE_LT(level, NumLevels()); + PADDLE_ENFORCE_LT(elem, NumElements(level)); return (lod_)[level][elem]; } @@ -82,12 +91,23 @@ class LoDTensor : public Tensor { * Number of elements in a level. */ size_t NumElements(size_t level = 0) const { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); + PADDLE_ENFORCE_LT(level, NumLevels()); // the last offset is the end of last element return (lod_)[level].size() - 1; } + /* + * Number of lower-level elements. + * For example, a 2-level lod-tensor + * + * 0-th level | | + * 1-th level || ||| + * + * NumElements(0, 0) get 2 + * NumElements(0, 1) get 3 + */ + size_t NumElements(size_t level, size_t idx) const; + /* * Shrink levels[level_begin:level_end] */ diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 486b839738ec077545163bc47e6a97ef188c3c2f..44f09f584fb752d7003baa804979f3bb5cd9d651 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -56,6 +56,12 @@ TEST_F(LoDTensorTester, NumElements) { ASSERT_EQ(lod_tensor_.NumElements(2), 8UL); } +TEST_F(LoDTensorTester, NumElements2) { + ASSERT_EQ(lod_tensor_.NumElements(0, 0), 2UL); + ASSERT_EQ(lod_tensor_.NumElements(0, 1), 2UL); + ASSERT_EQ(lod_tensor_.NumElements(1, 1), 2UL); +} + TEST_F(LoDTensorTester, ShrinkLevels) { // slice 1 level for (size_t level = 0; level < 3UL; ++level) { @@ -65,7 +71,7 @@ TEST_F(LoDTensorTester, ShrinkLevels) { ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level)); ASSERT_EQ(new_lod_tensor.data(), lod_tensor_.data()); } - // slice 2 level + // shrink 2 level for (size_t level = 0; level < 2UL; ++level) { LoDTensor new_lod_tensor = lod_tensor_; new_lod_tensor.ShrinkLevels(level, level + 2); diff --git a/paddle/framework/lod_tensor_test.cu b/paddle/framework/lod_tensor_test.cu index 97e69cdb2e5e1e64031c899f5e04020665485ba8..647d07536dd070bc37137fc01f683ec07ba7d6f4 100644 --- a/paddle/framework/lod_tensor_test.cu +++ b/paddle/framework/lod_tensor_test.cu @@ -36,8 +36,8 @@ TEST(LoDTensor, LoDInGPU) { lod_tensor.mutable_data(place); lod_tensor.set_lod(src_lod); - CHECK_EQ(lod_tensor.lod_element(0, 2), 4); - CHECK_EQ(lod_tensor.lod_element(0, 4), 8); + CHECK_EQ(lod_tensor.lod_element(0, 2), 4UL); + CHECK_EQ(lod_tensor.lod_element(0, 4), 8UL); auto lod = lod_tensor.lod(); diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc new file mode 100644 index 0000000000000000000000000000000000000000..99b5a9c37700adce56f9a83af3792ef113a873ff --- /dev/null +++ b/paddle/framework/op_desc.cc @@ -0,0 +1,133 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_desc.h" +#include "paddle/framework/block_desc.h" + +namespace paddle { +namespace framework { + +OpDesc *OpDescBind::Proto() { + Sync(); + return &op_desc_; +} + +const std::vector &OpDescBind::Input( + const std::string &name) const { + auto it = inputs_.find(name); + PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s", name, + Type()); + return it->second; +} + +std::vector OpDescBind::InputNames() const { + std::vector retv; + retv.reserve(this->inputs_.size()); + for (auto &ipt : this->inputs_) { + retv.push_back(ipt.first); + } + return retv; +} + +void OpDescBind::SetInput(const std::string ¶m_name, + const std::vector &args) { + need_update_ = true; + inputs_[param_name] = args; +} + +const std::vector &OpDescBind::Output( + const std::string &name) const { + auto it = outputs_.find(name); + PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s", + name, Type()); + return it->second; +} + +std::vector OpDescBind::OutputNames() const { + std::vector retv; + retv.reserve(this->outputs_.size()); + for (auto &ipt : this->outputs_) { + retv.push_back(ipt.first); + } + return retv; +} + +void OpDescBind::SetOutput(const std::string ¶m_name, + const std::vector &args) { + need_update_ = true; + this->outputs_[param_name] = args; +} + +AttrType OpDescBind::GetAttrType(const std::string &name) const { + auto it = attrs_.find(name); + PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); + return static_cast(it->second.which() - 1); +} + +std::vector OpDescBind::AttrNames() const { + std::vector retv; + retv.reserve(attrs_.size()); + for (auto &attr : attrs_) { + retv.push_back(attr.first); + } + return retv; +} + +void OpDescBind::SetAttr(const std::string &name, const Attribute &v) { + this->attrs_[name] = v; + need_update_ = true; +} + +Attribute OpDescBind::GetAttr(const std::string &name) const { + auto it = attrs_.find(name); + PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); + return it->second; +} + +int OpDescBind::GetBlockAttr(const std::string &name) const { + auto it = attrs_.find(name); + PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); + return boost::get(it->second)->idx(); +} + +void OpDescBind::Sync() { + if (need_update_) { + this->op_desc_.mutable_inputs()->Clear(); + for (auto &ipt : inputs_) { + auto *input = op_desc_.add_inputs(); + input->set_parameter(ipt.first); + VectorToRepeated(ipt.second, input->mutable_arguments()); + } + + this->op_desc_.mutable_outputs()->Clear(); + for (auto &opt : outputs_) { + auto *output = op_desc_.add_outputs(); + output->set_parameter(opt.first); + VectorToRepeated(opt.second, output->mutable_arguments()); + } + + this->op_desc_.mutable_attrs()->Clear(); + for (auto &attr : attrs_) { + auto *attr_desc = op_desc_.add_attrs(); + attr_desc->set_name(attr.first); + attr_desc->set_type( + static_cast(attr.second.which() - 1)); + boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second); + } + + need_update_ = false; + } +} +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h new file mode 100644 index 0000000000000000000000000000000000000000..ffc8ac61abfb74e4716f10c457d0fbc18b2e2ab8 --- /dev/null +++ b/paddle/framework/op_desc.h @@ -0,0 +1,106 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include "paddle/framework/attribute.h" +#include "paddle/framework/var_desc.h" + +namespace paddle { +namespace framework { + +class BlockDescBind; + +class OpDescBind { + public: + OpDesc *Proto(); + + std::string Type() const { return op_desc_.type(); } + + void SetType(const std::string &type) { op_desc_.set_type(type); } + + const std::vector &Input(const std::string &name) const; + + std::vector InputNames() const; + + void SetInput(const std::string ¶m_name, + const std::vector &args); + + const std::vector &Output(const std::string &name) const; + + std::vector OutputNames() const; + + void SetOutput(const std::string ¶m_name, + const std::vector &args); + + std::string DebugString() { return this->Proto()->DebugString(); } + + bool HasAttr(const std::string &name) const { + return attrs_.find(name) != attrs_.end(); + } + + AttrType GetAttrType(const std::string &name) const; + + std::vector AttrNames() const; + + void SetAttr(const std::string &name, const Attribute &v); + + void SetBlockAttr(const std::string &name, BlockDescBind &block); + + Attribute GetAttr(const std::string &name) const; + + int GetBlockAttr(const std::string &name) const; + + private: + struct SetAttrDescVisitor : public boost::static_visitor { + explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} + mutable OpDesc::Attr *attr_; + void operator()(int v) const { attr_->set_i(v); } + void operator()(float v) const { attr_->set_f(v); } + void operator()(const std::string &v) const { attr_->set_s(v); } + void operator()(bool b) const { attr_->set_b(b); } + + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_ints()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_floats()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_strings()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_bools()); + } + void operator()(BlockDesc *desc) const { + attr_->set_block_idx(desc->idx()); + } + void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } + }; + + void Sync(); + + OpDesc op_desc_; + std::unordered_map> inputs_; + std::unordered_map> outputs_; + std::unordered_map attrs_; + + // need_update_ indicate there some local changes not be synchronized. If + // local changes should be synchronized, need_update_ should be set to true. + bool need_update_{false}; +}; +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index e00c6e8d904508ec9985537fc703c7c61a14e0de..b6fc0409d5cb22b13352df41b8e911c79bc4825a 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -10,7 +10,6 @@ class CosineOp : public OperatorBase { using OperatorBase::OperatorBase; void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const override {} - void InferShape(const Scope& scope) const override {} }; class CosineOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { @@ -29,7 +28,6 @@ class CosineOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { class MyTestOp : public OperatorBase { public: using OperatorBase::OperatorBase; - void InferShape(const Scope& scope) const override {} void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const override {} }; @@ -174,4 +172,4 @@ TEST(OpRegistry, CustomChecker) { op->Run(scope, dev_ctx); int test_attr = op->Attr("test_attr"); ASSERT_EQ(test_attr, 4); -} \ No newline at end of file +} diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index fcbfc3e4377edd0ea55c8d4328c325fa18663001..d7beff5bc1df1def6bf35381e103cf87eeb68fd0 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -14,7 +14,7 @@ limitations under the License. */ #include "paddle/framework/operator.h" #include -#include "paddle/framework/op_registry.h" +#include namespace paddle { namespace framework { @@ -33,6 +33,24 @@ ExecutionContext::GetEigenDevice() const { } #endif +const Tensor* GetTensorFromVar(const Variable* var) { + if (var->IsType()) { + return &var->Get(); + } + PADDLE_ENFORCE(var->IsType(), + "The Input must be LoDTensor or Tensor."); + return &var->Get(); +} + +Tensor* GetTensorFromVar(Variable* var) { + if (var->IsType()) { + return var->GetMutable(); + } + PADDLE_ENFORCE(var->IsType(), + "The Input must be LoDTensor or Tensor."); + return var->GetMutable(); +} + std::string OperatorBase::Input(const std::string& name) const { auto& ins = Inputs(name); PADDLE_ENFORCE_LE(ins.size(), 1UL, diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 2d6d5510ef6dc83f1a016be6ff123f0b9bcaf230..79bda2e2f9173ab632307bc52167d7d8c17d4418 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include #include #include #include @@ -24,6 +25,7 @@ limitations under the License. */ #include "paddle/framework/framework.pb.h" #include "paddle/framework/lod_tensor.h" #include "paddle/framework/scope.h" +#include "paddle/framework/shape_inference.h" #include "paddle/framework/tensor.h" #include "paddle/platform/device_context.h" #include "paddle/platform/place.h" @@ -56,6 +58,9 @@ class OperatorBase; class InferShapeContext; class ExecutionContext; +extern const Tensor* GetTensorFromVar(const Variable* var); +extern Tensor* GetTensorFromVar(Variable* var); + /** * OperatorBase has the basic element that Net will call to do computation. * Only CreateOperator from OpRegistry will new Operator directly. User @@ -78,10 +83,6 @@ class OperatorBase { virtual std::string DebugString() const; - /// InferShape infer the size of Variables used by this Operator with - /// information inside scope - virtual void InferShape(const Scope& scope) const = 0; - /// Net will call this function to Run an op. virtual void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const = 0; @@ -159,7 +160,6 @@ class OperatorBase { class NOP : public OperatorBase { public: using OperatorBase::OperatorBase; - void InferShape(const Scope& scope) const override {} void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const override {} std::unique_ptr Clone() const override { @@ -262,15 +262,6 @@ class InferShapeContext { return res; } - const Tensor* GetTensorFromVar(const Variable* var) const { - if (var->IsType()) { - return &var->Get(); - } - PADDLE_ENFORCE(var->IsType(), - "The Input(%s) must be LoDTensor or Tensor."); - return &var->Get(); - } - void ShareLoD(const std::string& in, const std::string& out, size_t i = 0, size_t j = 0) const { PADDLE_ENFORCE_LT(i, InputSize(in)); @@ -340,6 +331,78 @@ class ExecutionContext : public InferShapeContext { const platform::DeviceContext& device_context_; }; +class RuntimeInferShapeContext : public InferShapeContextBase { + public: + RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope) + : op_(op), scope_(scope) {} + + bool HasInput(const std::string& name) const { + auto ipt = op_.Input(name); + auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); + return var != nullptr; + } + + bool HasOutput(const std::string& name) const { + auto ipt = op_.Output(name); + auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); + return var != nullptr; + } + + DDim GetInputDim(const std::string& name) const { + return GetDim(op_.Input(name)); + } + + void SetInputDim(const std::string& name, const DDim& dim) { + SetDim(op_.Input(name), dim); + } + + DDim GetOutputDim(const std::string& name) const { + return GetDim(op_.Output(name)); + } + + void SetOutputDim(const std::string& name, const DDim& dim) { + SetDim(op_.Output(name), dim); + } + + AttrReader Attrs() const { return AttrReader(op_.Attrs()); } + + const std::vector& Inputs(const std::string& name) const { + return op_.Inputs(name); + } + + const std::vector& Outputs(const std::string& name) const { + return op_.Outputs(name); + } + + private: + template + Tensor* GetTensor(const std::string& name) const { + Tensor* t = nullptr; + auto* var = scope_.FindVar(name); + if (!var->IsType() && !var->IsType()) { + if (Allocate) { + t = var->GetMutable(); + } else { + PADDLE_THROW("Variable(%s) should be tensor", name); + } + } else { + t = GetTensorFromVar(scope_.FindVar(name)); + } + return t; + } + + DDim GetDim(const std::string& name) const { + return GetTensor(name)->dims(); + } + + void SetDim(const std::string& name, const DDim& dim) { + GetTensor(name)->Resize(dim); + } + + const OperatorBase& op_; + const Scope& scope_; +}; + class OpKernel { public: /** @@ -383,12 +446,11 @@ class OperatorWithKernel : public OperatorBase { const VariableNameMap& outputs, const AttributeMap& attrs) : OperatorBase(type, inputs, outputs, attrs) {} - void InferShape(const Scope& scope) const override { - InferShape(InferShapeContext(*this, scope)); - } - void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const final { + RuntimeInferShapeContext infer_shape_ctx(*this, scope); + this->InferShape(&infer_shape_ctx); + auto& opKernel = AllOpKernels().at(type_).at(OpKernelKey(dev_ctx)); opKernel->Compute(ExecutionContext(*this, scope, dev_ctx)); } @@ -406,7 +468,7 @@ class OperatorWithKernel : public OperatorBase { } protected: - virtual void InferShape(const InferShapeContext& ctx) const = 0; + virtual void InferShape(InferShapeContextBase* ctx) const = 0; }; } // namespace framework diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 0beab0fac5b94c78121261d2661a6f969289afc4..e1d8f040b837a6ad598351dae0427cc7c231e79f 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/framework/operator.h" #include "gtest/gtest.h" +#include "paddle/framework/op_info.h" #include "paddle/framework/op_registry.h" namespace paddle { @@ -26,7 +27,6 @@ class OpWithoutKernelTest : public OperatorBase { OpWithoutKernelTest(const std::string& type, const VariableNameMap& inputs, const VariableNameMap& outputs, const AttributeMap& attrs) : OperatorBase(type, inputs, outputs, attrs), x(1) {} - void InferShape(const Scope& scope) const override {} void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const override { ++op_run_num; @@ -86,7 +86,6 @@ TEST(OperatorBase, all) { auto op = paddle::framework::OpRegistry::CreateOp(op_desc); scope.NewVar("OUT1"); ASSERT_EQ(paddle::framework::op_run_num, 0); - op->InferShape(scope); op->Run(scope, device_context); ASSERT_EQ(paddle::framework::op_run_num, 1); } @@ -114,7 +113,7 @@ class OpWithKernelTest : public OperatorWithKernel { using OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override {} + void InferShape(framework::InferShapeContextBase* ctx) const override {} }; template @@ -254,7 +253,6 @@ class OperatorClone : public paddle::framework::OperatorBase { const paddle::framework::VariableNameMap& outputs, const paddle::framework::AttributeMap& attrs) : OperatorBase(type, inputs, outputs, attrs) {} - void InferShape(const paddle::framework::Scope& scope) const override {} void Run(const paddle::framework::Scope& scope, const paddle::platform::DeviceContext& dev_ctx) const override {} }; diff --git a/paddle/framework/program_desc.cc b/paddle/framework/program_desc.cc new file mode 100644 index 0000000000000000000000000000000000000000..e89f9a46d587b6378aa3be92306c5680093e1926 --- /dev/null +++ b/paddle/framework/program_desc.cc @@ -0,0 +1,60 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/program_desc.h" +#include "paddle/framework/block_desc.h" + +namespace paddle { +namespace framework { + +using ProgDescMap = + std::unordered_map>; +static ProgDescMap *g_bind_map = nullptr; + +ProgramDescBind &ProgramDescBind::Instance(ProgramDesc *prog) { + if (g_bind_map == nullptr) { + g_bind_map = new ProgDescMap(); + } + auto &map = *g_bind_map; + auto &ptr = map[prog]; + + if (ptr == nullptr) { + ptr.reset(new ProgramDescBind(prog)); + } + return *ptr; +} + +BlockDescBind *ProgramDescBind::AppendBlock(const BlockDescBind &parent) { + auto *b = prog_->add_blocks(); + b->set_parent_idx(parent.ID()); + b->set_idx(prog_->blocks_size() - 1); + blocks_.emplace_back(new BlockDescBind(this, b)); + return blocks_.back().get(); +} + +ProgramDesc *ProgramDescBind::Proto() { + for (auto &block : blocks_) { + block->Sync(); + } + return prog_; +} + +ProgramDescBind::ProgramDescBind(ProgramDesc *prog) { + prog_ = prog; + for (auto &block : *prog->mutable_blocks()) { + blocks_.emplace_back(new BlockDescBind(this, &block)); + } +} +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h new file mode 100644 index 0000000000000000000000000000000000000000..06ffcd4b15078f62ea8b7a3714e73de799530785 --- /dev/null +++ b/paddle/framework/program_desc.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/framework/framework.pb.h" + +namespace paddle { +namespace framework { + +class BlockDescBind; + +class ProgramDescBind { + public: + static ProgramDescBind &Instance(ProgramDesc *prog); + + ProgramDescBind(const ProgramDescBind &o) = delete; + ProgramDescBind &operator=(const ProgramDescBind &o) = delete; + + BlockDescBind *AppendBlock(const BlockDescBind &parent); + + BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); } + + std::string DebugString() { return Proto()->DebugString(); } + + size_t Size() const { return blocks_.size(); } + + ProgramDesc *Proto(); + + private: + explicit ProgramDescBind(ProgramDesc *prog); + + // Not owned + ProgramDesc *prog_; + + std::vector> blocks_; +}; +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h new file mode 100644 index 0000000000000000000000000000000000000000..b07fc788124413f728c713027609d9d2d1c39538 --- /dev/null +++ b/paddle/framework/shape_inference.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/ddim.h" + +namespace paddle { +namespace framework { + +class InferShapeContextBase { + public: + virtual ~InferShapeContextBase() {} + virtual bool HasInput(const std::string &name) const = 0; + virtual bool HasOutput(const std::string &name) const = 0; + virtual framework::DDim GetInputDim(const std::string &name) const = 0; + std::vector GetInputsDim(const std::string &name) const { + const std::vector &names = Inputs(name); + return GetDims(names); + } + virtual void SetInputDim(const std::string &name, + const framework::DDim &dim) = 0; + void SetInputsDim(const std::string &name, + const std::vector &dims) { + auto &names = Inputs(name); + SetDims(names, dims); + } + virtual framework::DDim GetOutputDim(const std::string &name) const = 0; + std::vector GetOutputsDim(const std::string &name) const { + const std::vector &names = Outputs(name); + return GetDims(names); + } + virtual void SetOutputDim(const std::string &name, const DDim &dim) = 0; + void SetOutputsDim(const std::string &name, + const std::vector &dims) { + auto &names = Outputs(name); + SetDims(names, dims); + } + virtual AttrReader Attrs() const = 0; + virtual const std::vector &Inputs( + const std::string &name) const = 0; + virtual const std::vector &Outputs( + const std::string &name) const = 0; + // TODO(qiao) implement this function + void ShareLoD(const std::string &in, const std::string &out, size_t i = 0, + size_t j = 0) const {} + + protected: + virtual framework::DDim GetDim(const std::string &name) const = 0; + virtual void SetDim(const std::string &name, const framework::DDim &dim) = 0; + std::vector GetDims( + const std::vector &names) const { + std::vector ret; + ret.reserve(names.size()); + std::transform( + names.begin(), names.end(), std::back_inserter(ret), + [this](const std::string &name) { return this->GetDim(name); }); + return ret; + } + void SetDims(const std::vector &names, + const std::vector &dims) { + size_t length = names.size(); + PADDLE_ENFORCE_EQ(length, dims.size()); + for (size_t i = 0; i < length; ++i) { + SetDim(names[i], dims[i]); + } + } +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc new file mode 100644 index 0000000000000000000000000000000000000000..13b9c5f3cdf98e6d22f4217fa1cf9a48910a78d8 --- /dev/null +++ b/paddle/framework/var_desc.cc @@ -0,0 +1,36 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/var_desc.h" + +namespace paddle { +namespace framework { + +void VarDescBind::SetShape(const std::vector &dims) { + VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims()); +} + +void VarDescBind::SetDataType(DataType data_type) { + desc_.mutable_lod_tensor()->set_data_type(data_type); +} + +std::vector VarDescBind::Shape() const { + return RepeatedToVector(desc_.lod_tensor().dims()); +} + +DataType VarDescBind::GetDataType() const { + return desc_.lod_tensor().data_type(); +} +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h new file mode 100644 index 0000000000000000000000000000000000000000..4763bf09d004539ab24e4aad3bf429667f1fcc73 --- /dev/null +++ b/paddle/framework/var_desc.h @@ -0,0 +1,73 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include "paddle/framework/framework.pb.h" + +namespace paddle { +namespace framework { + +// convert between std::vector and protobuf repeated. +template +inline std::vector RepeatedToVector( + const google::protobuf::RepeatedField &repeated_field) { + std::vector ret; + ret.reserve(repeated_field.size()); + std::copy(repeated_field.begin(), repeated_field.end(), + std::back_inserter(ret)); + return ret; +} + +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Reserve(vec.size()); + for (const auto &elem : vec) { + *repeated_field->Add() = elem; + } +} + +// Specialize vector. +template +inline void VectorToRepeated(const std::vector &vec, + RepeatedField *repeated_field) { + repeated_field->Reserve(vec.size()); + for (auto elem : vec) { + *repeated_field->Add() = elem; + } +} + +class VarDescBind { + public: + explicit VarDescBind(const std::string &name) { desc_.set_name(name); } + + VarDesc *Proto() { return &desc_; } + + std::string Name() const { return desc_.name(); } + + void SetShape(const std::vector &dims); + + void SetDataType(DataType data_type); + + std::vector Shape() const; + + DataType GetDataType() const; + + private: + VarDesc desc_; +}; +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/variable.md b/paddle/framework/variable.md index f44d5ea46e7ce98dd443d684ad42308496bc4179..442ef6b718b227d79ca73031efcbb55817558252 100644 --- a/paddle/framework/variable.md +++ b/paddle/framework/variable.md @@ -7,7 +7,7 @@ Variable is also known as *blob* in MxNet and Caffe2. It is the input and outpu For the flexibility of a DL system, a variable should be able to contain any typed value -- a tensor in most cases, but could also be some integer IDs or a scope of other variables in the case of RNN. -To use the minimum amount of memory, we'd like that a variable to allocate memory when it has to, or, lazy memory allocation. Let's take the following example: +To use the minimum amount of memory, we would like that a variable allocates memory only when it has to, or, lazy memory allocation. Let's take the following example: ```cpp Variable vr, v1, v2; @@ -38,7 +38,7 @@ This syntax for lazy memory allocation when we call `Randomize` and `Mult`, thos To make memory allocation lazy, we cannot assume that we know the type held by a variable at definition time. In other words, `class Variable` cannot be a template `template class Variable`. -Because we don't know the type `T`, we cannot save a `T*` as `Variable's` data member. Instead, we save an interface object `Placeholder`, who can return the pointer to the saved object via `Placeholder::Ptr()` as `void*`. +Because we don't know the type `T`, we cannot save a `T*` as `Variable's` data member. Instead, we save an interface object `Placeholder`, which can return the pointer to the saved object via `Placeholder::Ptr()` as `void*`. But anyway, Variable needs to know `T` so could it `delete(ptr)` and so could `Variable::Get` checks the expected type and the saved object's type. @@ -49,4 +49,4 @@ Because `PlaceholderImpl` knows `T`, it can save and return `typeid(T)` for the ## Conclusion -The technique type hiding utilizes C++ class templates, interface and derivation, and C++ RTTI (typeid). This combination saves us from definition something like `caffe2::TypeMata`, which takes hundreds of lines of C++ code. +The technique type hiding utilizes C++ class templates, interface and derivation, and C++ RTTI (typeid). This combination saves us from defining something like `caffe2::TypeMeta`, which takes hundreds of lines of C++ code. diff --git a/paddle/function/neon/NeonDepthwiseConv.h b/paddle/function/neon/NeonDepthwiseConv.h index 33722d3cac61b62f5dce8f51105c1bf4e70c4a6c..98a86d278f39e70472793e6a1d38f7dae469fd62 100644 --- a/paddle/function/neon/NeonDepthwiseConv.h +++ b/paddle/function/neon/NeonDepthwiseConv.h @@ -18,7 +18,6 @@ limitations under the License. */ #include "neon_util.h" namespace paddle { - namespace neon { #if defined(__ARM_NEON__) || defined(__ARM_NEON) @@ -26,17 +25,20 @@ namespace neon { template struct DepthwiseConvKernel {}; -inline float32_t conv3x3(float32x4_t r0, - float32x4_t r1, - float32x4_t r2, +inline float32_t conv3x3(const float* r0, + const float* r1, + const float* r2, float32x4_t k0, float32x4_t k1, float32x4_t k2) { - float32x4_t tmp; - tmp = vmulq_f32(r0, k0); - tmp = vmlaq_f32(tmp, r1, k1); - tmp = vmlaq_f32(tmp, r2, k2); - return vaddvq_f32(tmp); + float32_t tmp[12]; + vst1q_f32(&(tmp[0]), k0); + vst1q_f32(&(tmp[4]), k1); + vst1q_f32(&(tmp[8]), k2); + float32_t sum0 = r0[0] * tmp[0] + r0[1] * tmp[1] + r0[2] * tmp[2]; + float32_t sum1 = r1[0] * tmp[4] + r1[1] * tmp[5] + r1[2] * tmp[6]; + float32_t sum2 = r2[0] * tmp[8] + r2[1] * tmp[9] + r2[2] * tmp[10]; + return sum0 + sum1 + sum2; } inline float32_t conv4x4(float32x4_t r0, @@ -136,10 +138,7 @@ struct DepthwiseConvKernel<3, 1> { } for (int r = 0; r < remain; r++) { - float32x4_t i0 = vld1q_f32(r0); - float32x4_t i1 = vld1q_f32(r1); - float32x4_t i2 = vld1q_f32(r2); - *outputData = conv3x3(i0, i1, i2, k[0], k[1], k[2]); + *outputData = conv3x3(r0, r1, r2, k[0], k[1], k[2]); r0++; r1++; r2++; @@ -243,10 +242,7 @@ struct DepthwiseConvKernel<3, 2> { } for (int r = 0; r < remain; r++) { - float32x4_t i0 = vld1q_f32(r0); - float32x4_t i1 = vld1q_f32(r1); - float32x4_t i2 = vld1q_f32(r2); - *outputData = conv3x3(i0, i1, i2, k[0], k[1], k[2]); + *outputData = conv3x3(r0, r1, r2, k[0], k[1], k[2]); r0 += 2; r1 += 2; r2 += 2; diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index 9a0abd291ae8fae43b0e95c7371f3ce35d1261ec..0d6742e909635c1097b4fe21bbb304f8a71af5cb 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -28,7 +28,7 @@ bool MKLDNNConvLayer::init(const LayerMap& layerMap, if (!MKLDNNLayer::init(layerMap, parameterMap)) { return false; } - CHECK_EQ(inputLayers_.size(), 1) << "Only support one input layer yet"; + CHECK_EQ(inputLayers_.size(), 1UL) << "Only support one input layer yet"; CHECK_EQ(inputLayers_.size(), parameters_.size()); CHECK(config_.shared_biases()) << "Only support shared biases yet"; diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index 8cbfbd0d2b9f2149f7c959aec5c4ae1de952f903..e829456d6afd7cc844f752d4571cd9f90c73997f 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -28,7 +28,7 @@ bool MKLDNNFcLayer::init(const LayerMap& layerMap, return false; } - CHECK_EQ(inputLayers_.size(), 1) << "Only support one input layer yet"; + CHECK_EQ(inputLayers_.size(), 1UL) << "Only support one input layer yet"; CHECK_EQ(inputLayers_.size(), parameters_.size()); CHECK(!parameters_[0]->isSparse()) << "Do not support sparse yet"; diff --git a/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp b/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp index 538d18cdc3d262df0ddb031d9e6b38a3fea57606..c922237d33da5de0ece61df732334bee5592249d 100644 --- a/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp +++ b/paddle/gserver/tests/test_CrossEntropyOverBeamGrad.cpp @@ -228,7 +228,7 @@ void genGroundTruth(vector& beamExpansions, curBeam.groundTruth[j] = *(start + n); curBeam.inBeam[j] = 1; } else { - CHECK_LE(curBeam.rowIdxInBeam[j] + 1, + CHECK_LE((size_t)curBeam.rowIdxInBeam[j] + 1, curBeam.subSeqStartPos.size() - 1); int start = curBeam.subSeqStartPos[curBeam.rowIdxInBeam[j]]; int end = curBeam.subSeqStartPos[curBeam.rowIdxInBeam[j] + 1]; diff --git a/paddle/memory/.clang-format b/paddle/memory/.clang-format deleted file mode 100644 index 29282dc87e2c499988c17d90d47d44cd5cf7f115..0000000000000000000000000000000000000000 --- a/paddle/memory/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/memory/.clang-format b/paddle/memory/.clang-format new file mode 120000 index 0000000000000000000000000000000000000000..7d28cb3924707d39dafe20f4664fb17b5538996c --- /dev/null +++ b/paddle/memory/.clang-format @@ -0,0 +1 @@ +../framework/.clang-format \ No newline at end of file diff --git a/paddle/memory/memcpy.cc b/paddle/memory/memcpy.cc index 19ec9ba9b26f5919796181a19a048b7edb508bdd..c96a697a7e022684688b31c05da43e52812100d8 100644 --- a/paddle/memory/memcpy.cc +++ b/paddle/memory/memcpy.cc @@ -80,6 +80,15 @@ void Copy(platform::GPUPlace dst_place, platform::GpuMemcpySync(dst, src, num, cudaMemcpyHostToDevice); } +template <> +void Copy(platform::GPUPlace dst_place, + void* dst, + platform::GPUPlace src_place, + const void* src, size_t num) { + platform::SetDeviceId(dst_place.device); + platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToDevice); +} + #endif // PADDLE_ONLY_CPU } // namespace memory diff --git a/paddle/operators/.clang-format b/paddle/operators/.clang-format deleted file mode 100644 index 47b8a85206ab457e2b3cb90a68b7a82a0753d327..0000000000000000000000000000000000000000 --- a/paddle/operators/.clang-format +++ /dev/null @@ -1,5 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -Standard: Cpp11 -... diff --git a/paddle/operators/.clang-format b/paddle/operators/.clang-format new file mode 120000 index 0000000000000000000000000000000000000000..7d28cb3924707d39dafe20f4664fb17b5538996c --- /dev/null +++ b/paddle/operators/.clang-format @@ -0,0 +1 @@ +../framework/.clang-format \ No newline at end of file diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index f8b0bce6815ff17a60ef64b0eec34a7cc9d16e72..e56895c63a426b782f7b46091bc86c367d49899d 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -88,10 +88,14 @@ add_subdirectory(math) set(DEPS_OPS recurrent_op - cond_op) + cond_op + cross_entropy_op + softmax_with_cross_entropy_op) op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc DEPS framework_proto tensor net_op) op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op) +op_library(cross_entropy_op DEPS cross_entropy_function) +op_library(softmax_with_cross_entropy_op DEPS cross_entropy_function softmax_function) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc index 70e4f9da1221ab300e2b507a3da2f7c5da93f2e4..82010bfb53e58a0836c99c353590f4e32e25ac4a 100644 --- a/paddle/operators/accuracy_op.cc +++ b/paddle/operators/accuracy_op.cc @@ -22,25 +22,23 @@ class AccuracyOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("Inference"), - "Input(Inference) of AccuracyOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), - "Input(Label) of AccuracyOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Accuracy"), - "Output(Accuracy) of AccuracyOp should not be null."); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Inference"), + "Input(Inference) of AccuracyOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), + "Input(Label) of AccuracyOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Accuracy"), + "Output(Accuracy) of AccuracyOp should not be null."); - auto *inference = ctx.Input("Inference"); - auto *label = ctx.Input("Label"); + auto inference_dim = ctx->GetInputDim("Inference"); + auto label_dim = ctx->GetInputDim("Label"); - PADDLE_ENFORCE_EQ(label->dims().size(), 1, "label must be a vector"); - PADDLE_ENFORCE_EQ(inference->dims()[0], label->dims()[0], + PADDLE_ENFORCE_EQ(label_dim.size(), 1, "label must be a vector"); + PADDLE_ENFORCE_EQ(inference_dim[0], label_dim[0], "inference size must be the same as label size"); - ctx.Output("Accuracy")->Resize({1}); - ctx.ShareLoD("Inference", /*->*/ "Accuracy"); + ctx->SetOutputDim("Accuracy", {1}); + ctx->ShareLoD("Inference", /*->*/ "Accuracy"); } }; diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 06654702bc42cc7cf4917b00693334b1d36ce371..f77e1c572e33533ac672e3d476a7e6dad122031f 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -22,10 +22,9 @@ class ActivationOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); - ctx.ShareLoD("X", /*->*/ "Y"); + void InferShape(framework::InferShapeContextBase *ctx) const override { + ctx->SetOutputDim("Y", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Y"); } }; @@ -34,9 +33,8 @@ class ActivationOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("Y")->dims()); + void InferShape(framework::InferShapeContextBase *ctx) const override { + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("Y")); } }; diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc index ed11d096974341022637676537793645f46738f0..3914d1323083ede6a7ea07e7b4ef76b9e4afd26d 100644 --- a/paddle/operators/add_op.cc +++ b/paddle/operators/add_op.cc @@ -22,25 +22,23 @@ class AddOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of AddOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of AddOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of AddOp should not be null."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of AddOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) of AddOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of AddOp should not be null."); - PADDLE_ENFORCE_EQ(ctx.Input("X")->dims(), - ctx.Input("Y")->dims(), + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + PADDLE_ENFORCE_EQ(x_dims, y_dims, "Two input of Add Op's dimension must be same."); - ctx.Output("Out")->Resize( - ctx.Input("X")->dims()); + ctx->SetOutputDim("Out", x_dims); } }; class AddOpMaker : public framework::OpProtoAndCheckerMaker { public: - AddOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + AddOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The first input of add op"); AddInput("Y", "The second input of add op"); @@ -58,7 +56,7 @@ class AddOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override {} + void InferShape(framework::InferShapeContextBase* ctx) const override {} }; } // namespace operators diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc index e5a54bc4b226fd24337050fdd84b2de9c49f7949..b3dd060fd725fc9056b25e4affd82fdb345e77f7 100644 --- a/paddle/operators/clip_op.cc +++ b/paddle/operators/clip_op.cc @@ -22,28 +22,28 @@ class ClipOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of ClipOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of ClipOp should not be null."); - auto x_dims = ctx.Input("X")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of ClipOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ClipOp should not be null."); + auto x_dims = ctx->GetInputDim("X"); auto max = Attr("max"); auto min = Attr("min"); PADDLE_ENFORCE_LT(min, max, "max should be greater than min."); - ctx.Output("Out")->Resize(x_dims); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", /*->*/ "Out"); } }; template class ClipOpMaker : public framework::OpProtoAndCheckerMaker { public: - ClipOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + ClipOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor)The input of clip op." - "The input should be a k-D tensor(k > 0 and k < 7)"); + "The number of dimensions must be between [1, 9]."); AddOutput("Out", "(Tensor)The output of clip op with shape as input(X)"); AddAttr( "min", "(float)Minimum value, under which element is replaced by min."); @@ -61,14 +61,13 @@ class ClipOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto *x_grad = ctx.Output(framework::GradVarName("X")); - if (x_grad != nullptr) { - x_grad->Resize(x_dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + if (ctx->HasOutput(framework::GradVarName("X"))) { + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); } } }; diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc index 07f847079e834716904dcc038d2097efd268bd3e..1ffa02c8f94c01a385d3ba376c1fd0dc3c1bd372 100644 --- a/paddle/operators/concat_op.cc +++ b/paddle/operators/concat_op.cc @@ -24,31 +24,32 @@ class ConcatOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of ConcatOp should not be null."); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE_GE(ctx->Inputs("X").size(), 1UL, + "Inputs(X) of ConcatOp should be empty.") + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ConcatOp should not be null."); - auto ins = ctx.MultiInput("X"); - auto *out = ctx.Output("Out"); - size_t axis = static_cast(ctx.Attr("axis")); - size_t n = ins.size(); + auto ins = ctx->GetInputsDim("X"); + size_t axis = static_cast(ctx->Attrs().Get("axis")); + const size_t n = ins.size(); PADDLE_ENFORCE_GT(n, 1, "Input tensors count should > 1."); - auto out_dims = ins[0]->dims(); + auto out_dims = ins[0]; size_t in_zero_dims_size = out_dims.size(); for (size_t i = 1; i < n; i++) { for (size_t j = 0; j < in_zero_dims_size; j++) { if (j == axis) { - out_dims[axis] += ins[i]->dims()[j]; + out_dims[axis] += ins[i][j]; continue; } - PADDLE_ENFORCE_EQ(out_dims[j], ins[i]->dims()[j], + PADDLE_ENFORCE_EQ(out_dims[j], ins[i][j], "Input tensors should have the same " "elements except the specify axis.") } } - out->Resize(out_dims); + ctx->SetOutputDim("Out", out_dims); } }; @@ -73,10 +74,27 @@ class ConcatOpMaker : public framework::OpProtoAndCheckerMaker { } }; +class ConcatOpGrad : public framework::OperatorWithKernel { + public: + ConcatOpGrad(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(framework::InferShapeContextBase *ctx) const override { + ctx->SetOutputsDim(framework::GradVarName("X"), ctx->GetInputsDim("X")); + } +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(concat, ops::ConcatOp, ops::ConcatOpMaker) +REGISTER_OP(concat, ops::ConcatOp, ops::ConcatOpMaker, concat_grad, + ops::ConcatOpGrad) REGISTER_OP_CPU_KERNEL(concat, ops::ConcatKernel) +REGISTER_OP_CPU_KERNEL(concat_grad, + ops::ConcatGradKernel) diff --git a/paddle/operators/concat_op.cu b/paddle/operators/concat_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..ede832ddcd486729db56bba016683b33875f8837 --- /dev/null +++ b/paddle/operators/concat_op.cu @@ -0,0 +1,20 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/concat_op.h" +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(concat, + ops::ConcatKernel); +REGISTER_OP_GPU_KERNEL( + concat_grad, ops::ConcatGradKernel); diff --git a/paddle/operators/concat_op.h b/paddle/operators/concat_op.h index f977054fdf8aa0164db726b94a21c57f770dd674..b37063261123bce1f22c39ab021e88f2faf58e9f 100644 --- a/paddle/operators/concat_op.h +++ b/paddle/operators/concat_op.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include "paddle/framework/op_registry.h" +#include "paddle/operators/strided_memcpy.h" namespace paddle { namespace operators { @@ -27,35 +28,39 @@ class ConcatKernel : public framework::OpKernel { auto ins = ctx.MultiInput("X"); auto* out = ctx.Output("Out"); int64_t axis = static_cast(ctx.Attr("axis")); - size_t n = ins.size(); - size_t output_axis_dim = 0; - size_t before = 1, after = 1; - for (size_t i = 0; i < n; i++) { - output_axis_dim += ins[i]->dims()[axis]; - } - auto& input_zero = ins[0]; - for (int64_t i = 0; i < input_zero->dims().size(); i++) { - if (i == axis) { - continue; - } - if (i < axis) { - before *= input_zero->dims()[i]; - } else { - after *= input_zero->dims()[i]; - } - } + const size_t n = ins.size(); size_t output_offset = 0; + out->mutable_data(ctx.GetPlace()); + auto out_stride = framework::stride(out->dims()); for (size_t i = 0; i < n; i++) { auto& in = ins[i]; auto axis_dim = in->dims()[axis]; - for (size_t j = 0; j < before; j++) { - size_t len = axis_dim * after * sizeof(T); - const T* src = in->data() + axis_dim * after * j; - T* out_data = out->mutable_data(platform::CPUPlace()); - T* dest = out_data + output_offset + output_axis_dim * after * j; - memcpy(dest, src, len); - } - output_offset += axis_dim * after; + auto in_stride = framework::stride(in->dims()); + StridedMemcpy(ctx.device_context(), in->data(), in_stride, + in->dims(), out_stride, out->data() + output_offset); + output_offset += axis_dim * in_stride[axis]; + } + } +}; + +template +class ConcatGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* in = ctx.Input(framework::GradVarName("Out")); + auto outs = ctx.MultiOutput(framework::GradVarName("X")); + int64_t axis = static_cast(ctx.Attr("axis")); + const size_t n = outs.size(); + size_t input_offset = 0; + auto in_stride = framework::stride(in->dims()); + for (size_t i = 0; i < n; i++) { + auto& out = outs[i]; + out->mutable_data(ctx.GetPlace()); + size_t axis_dim = out->dims()[axis]; + auto out_stride = framework::stride(out->dims()); + StridedMemcpy(ctx.device_context(), in->data() + input_offset, + in_stride, out->dims(), out_stride, out->data()); + input_offset += axis_dim * in_stride[axis]; } } }; diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index 8262a7a5c8c13c86c5f6c123a14fa89696358c57..aaffa6661fe4686d09f20f0f0682219772638202 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -82,7 +82,7 @@ void CondOp::InferShape(const Scope& scope) const { } // each net calls InferShape - sub_net_op_[i]->InferShape(*sub_scopes[i]); + // sub_net_op_[i]->InferShape(*sub_scopes[i]); } for (auto& output : Outputs("Outs")) { @@ -215,7 +215,7 @@ class CondOpProtoAndCheckerMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Sample dependent Cond Operator: Given Cond[i] as a 1/0 vector to indicate true/false -The equation is: +The equation is: Out[i] = subnet_t[i], if Cond[i] == true Out[i] = subnet_t[i], if Cond[i] == false )DOC"); diff --git a/paddle/operators/cond_op.h b/paddle/operators/cond_op.h index b09e32331e66c53555c88c06d7b1456276050eaa..9a88ee35f108204348baddc57e0c0d8e63c3fb6d 100644 --- a/paddle/operators/cond_op.h +++ b/paddle/operators/cond_op.h @@ -57,8 +57,10 @@ class CondOp : public framework::OperatorBase { /* * InferShape must be called before Run. + * FIXME(yuyang18): Since InferShape has been removed, this implementation + * could be wrong. */ - void InferShape(const framework::Scope& scope) const override; + void InferShape(const framework::Scope& scope) const; /* * Set True Block diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc index c3281db0964de6d7dd6be629fbcc55cabb9fef9d..5cc82944bb6b9a4fc5cd94cf2233ab84fc105fe7 100644 --- a/paddle/operators/conv2d_op.cc +++ b/paddle/operators/conv2d_op.cc @@ -27,27 +27,25 @@ class Conv2DOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Input"), - "Input(Input) of Conv2DOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Filter"), - "Input(Filter) of Conv2DOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Output"), - "Output(Output) of Conv2DOp should not be null."); - - auto in = ctx.Input("Input"); - auto filter = ctx.Input("Filter"); - auto out = ctx.Output("Output"); - std::vector strides = Attr>("strides"); - std::vector paddings = Attr>("paddings"); - int groups = Attr("groups"); - int input_channels = in->dims()[1]; - int output_channels = filter->dims()[0]; - - PADDLE_ENFORCE_EQ(in->dims().size(), 4, "Conv2DOp input should be 4-D."); - PADDLE_ENFORCE_EQ(filter->dims().size(), 4, - "Conv2DOp filter should be 4-D."); - PADDLE_ENFORCE_EQ(input_channels, filter->dims()[1] * groups, + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(Input) of Conv2DOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Filter"), + "Input(Filter) of Conv2DOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Output"), + "Output(Output) of Conv2DOp should not be null."); + + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + std::vector strides = ctx->Attrs().Get>("strides"); + std::vector paddings = ctx->Attrs().Get>("paddings"); + int groups = ctx->Attrs().Get("groups"); + int input_channels = in_dims[1]; + int output_channels = filter_dims[0]; + + PADDLE_ENFORCE_EQ(in_dims.size(), 4, "Conv2DOp input should be 4-D."); + PADDLE_ENFORCE_EQ(filter_dims.size(), 4, "Conv2DOp filter should be 4-D."); + PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups, "The number of input channels should be equal to filter " "channels * groups."); PADDLE_ENFORCE_EQ( @@ -55,17 +53,17 @@ class Conv2DOp : public framework::OperatorWithKernel { "The number of output channels should be divided by groups."); auto output_height = - outputSize(in->dims()[2], filter->dims()[2], paddings[0], strides[0]); + outputSize(in_dims[2], filter_dims[2], paddings[0], strides[0]); auto output_width = - outputSize(in->dims()[3], filter->dims()[3], paddings[1], strides[1]); - out->Resize( - {in->dims()[0], filter->dims()[0], output_height, output_width}); + outputSize(in_dims[3], filter_dims[3], paddings[1], strides[1]); + ctx->SetOutputDim( + "Output", {in_dims[0], filter_dims[0], output_height, output_width}); } }; class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker { public: - Conv2DOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + Conv2DOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "Input", @@ -108,14 +106,15 @@ class Conv2DOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - auto in = ctx.Input("Input"); - auto filter = ctx.Input("Filter"); - auto d_in = ctx.Output(framework::GradVarName("Input")); - auto d_filter = - ctx.Output(framework::GradVarName("Filter")); - if (d_in) d_in->Resize(in->dims()); - if (d_filter) d_filter->Resize(filter->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + if (ctx->HasOutput(framework::GradVarName("Input"))) { + ctx->SetOutputDim(framework::GradVarName("Input"), in_dims); + } + if (ctx->HasOutput(framework::GradVarName("Filter"))) { + ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims); + } } }; diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc index b56ee2047b811e212b4bf74bf7fbba753a6bcb11..040546f1a6fe1af6d17a5e363a11d27de88d03c2 100644 --- a/paddle/operators/cos_sim_op.cc +++ b/paddle/operators/cos_sim_op.cc @@ -24,22 +24,22 @@ class CosSimOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { + void InferShape(framework::InferShapeContextBase* ctx) const override { // notnull check - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of CosSimOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of CosSimOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of CosSimOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("XNorm"), - "Output(XNorm) of CosSimOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("YNorm"), - "Output(YNorm) of CosSimOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of CosSimOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), + "Input(Y) of CosSimOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of CosSimOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("XNorm"), + "Output(XNorm) of CosSimOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("YNorm"), + "Output(YNorm) of CosSimOp should not be null."); // shape check - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); PADDLE_ENFORCE_EQ(x_dims.size(), y_dims.size(), "Ranks of Input(X) and Input(Y) must be equal."); @@ -54,16 +54,16 @@ class CosSimOp : public framework::OperatorWithKernel { " just 1 (which will be broadcasted to match Input(X))."); // resize tensor - ctx.Output("Out")->Resize({x_dims[0], 1}); - ctx.Output("XNorm")->Resize({x_dims[0], 1}); - ctx.Output("YNorm")->Resize({y_dims[0], 1}); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", {x_dims[0], 1}); + ctx->SetOutputDim("XNorm", {x_dims[0], 1}); + ctx->SetOutputDim("YNorm", {y_dims[0], 1}); + ctx->ShareLoD("X", /*->*/ "Out"); } }; class CosSimOpMaker : public framework::OpProtoAndCheckerMaker { public: - CosSimOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + CosSimOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The 1st input of cos_sim op."); AddInput("Y", "The 2nd input of cos_sim op."); @@ -98,27 +98,23 @@ class CosSimOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { + void InferShape(framework::InferShapeContextBase* ctx) const override { // notnull check - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("XNorm"), - "Input(XNorm) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("YNorm"), - "Input(YNorm) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Out"), - "Input(Out) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("XNorm"), "Input(XNorm) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("YNorm"), "Input(YNorm) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("Out"), "Input(Out) must not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) must not be null."); // shape check - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - auto xnorm_dims = ctx.Input("XNorm")->dims(); - auto ynorm_dims = ctx.Input("YNorm")->dims(); - auto out_dims = ctx.Input("Out")->dims(); - auto out_grad_dims = - ctx.Input(framework::GradVarName("Out"))->dims(); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto xnorm_dims = ctx->GetInputDim("XNorm"); + auto ynorm_dims = ctx->GetInputDim("YNorm"); + auto out_dims = ctx->GetInputDim("Out"); + auto out_grad_dims = ctx->GetInputDim(framework::GradVarName("Out")); PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), "Ranks of Input(X) and Input(Y) must be equal."); @@ -143,10 +139,14 @@ class CosSimOpGrad : public framework::OperatorWithKernel { "Shape of Input(Out@Grad) must be [X.Dim(0), 1]."); // resize tensor - auto *x_grad = ctx.Output(framework::GradVarName("X")); - auto *y_grad = ctx.Output(framework::GradVarName("Y")); - if (x_grad) x_grad->Resize(x_dims); - if (y_grad) y_grad->Resize(y_dims); + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); + } } }; diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc index 52a1123348b10e39bcfa1ba062c893e5f20ed862..9b2305e90e85a6f39d4c584a3251b25f67e81aca 100644 --- a/paddle/operators/crop_op.cc +++ b/paddle/operators/crop_op.cc @@ -25,16 +25,14 @@ class CropOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of CropOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of CropOp should not be null."); - auto x_dim = ctx.Input("X")->dims(); - auto *y = ctx.Input("Y"); - auto *out = ctx.Output("Out"); - if (y == nullptr) { - auto shape = Attr>("shape"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of CropOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of CropOp should not be null."); + auto x_dim = ctx->GetInputDim("X"); + if (!ctx->HasInput("Y")) { + auto shape = ctx->Attrs().Get>("shape"); PADDLE_ENFORCE_EQ( int64_t(shape.size()), x_dim.size(), "Shape size should be equal to dimention size of input tensor."); @@ -42,19 +40,20 @@ class CropOp : public framework::OperatorWithKernel { for (size_t i = 0; i < shape.size(); ++i) { tensor_shape[i] = static_cast(shape[i]); } - out->Resize(framework::make_ddim(tensor_shape)); + ctx->SetOutputDim("Out", framework::make_ddim(tensor_shape)); } else { - PADDLE_ENFORCE_EQ(framework::arity(x_dim), framework::arity(y->dims()), + auto y_dim = ctx->GetInputDim("Y"); + PADDLE_ENFORCE_EQ(framework::arity(x_dim), framework::arity(y_dim), "Tensor rank of both CropOp's " "inputs must be same."); - out->Resize(y->dims()); + ctx->SetOutputDim("Out", y_dim); } } }; class CropOpMaker : public framework::OpProtoAndCheckerMaker { public: - CropOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + CropOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of pad op. " @@ -78,12 +77,12 @@ class CropOpMaker : public framework::OpProtoAndCheckerMaker { Crop Operator. Crop input into output, as specified by offsets and shape. -There are two ways to set shape: +There are two ways to set shape: 1. referenc input: crop input X as shape as reference input. - The dimension of reference input should + The dimension of reference input should be as same as input X. 2. shape list: crop input X by shape described by a list. - The size of shape list should be as same as + The size of shape list should be as same as dimension size of input X. The input should be a k-D tensor(k > 0 and k < 7). As an example: @@ -94,15 +93,15 @@ Given: [0, 3, 4, 0, 0] [0, 0, 0, 0, 0]] -and +and offsets = [0, 1] and - + shape = [2, 2] -then we get +then we get Out = [[1, 2], [3, 4]] @@ -116,14 +115,14 @@ class CropOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto *x_grad = ctx.Output(framework::GradVarName("X")); - if (x_grad != nullptr) { - x_grad->Resize(x_dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + auto x_grad_name = framework::GradVarName("X"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); } } }; diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index 2e16201e74c153888594ebe6679fb0036734dad4..26fc9b51c44d21d92851030449e116538f937846 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -22,33 +22,30 @@ class CrossEntropyOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), - "Input(Label) should be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) should be not null."); - - auto x = ctx.Input("X"); - auto label = ctx.Input("Label"); - PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank should be 2."); - PADDLE_ENFORCE_EQ(label->dims().size(), 2, - "Input(Label)'s rank should be 2."); - PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("Y"), "Output(Y) should be not null."); + + auto x_dims = ctx->GetInputDim("X"); + auto label_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank should be 2."); + PADDLE_ENFORCE_EQ(label_dims.size(), 2, "Input(Label)'s rank should be 2."); + PADDLE_ENFORCE_EQ(x_dims[0], label_dims[0], "The 1st dimension of Input(X) and Input(Label) should " "be equal."); - if (ctx.Attr("softLabel")) { - PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], + if (ctx->Attrs().Get("softLabel")) { + PADDLE_ENFORCE_EQ(x_dims[1], label_dims[1], "If Attr(softLabel) == true, the 2nd dimension of " "Input(X) and Input(Label) should be equal."); } else { - PADDLE_ENFORCE_EQ(label->dims()[1], 1, + PADDLE_ENFORCE_EQ(label_dims[1], 1, "If Attr(softLabel) == false, the 2nd dimension of " "Input(Label) should be 1."); } - ctx.Output("Y")->Resize({x->dims()[0], 1}); - ctx.ShareLoD("X", /*->*/ "Y"); + ctx->SetOutputDim("Y", {x_dims[0], 1}); + ctx->ShareLoD("X", /*->*/ "Y"); } }; @@ -57,50 +54,45 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), - "Input(Label) should be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Y")), - "Input(Y@GRAD) shoudl be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar(framework::GradVarName("X")), - "Output(X@GRAD) should be not null."); - - auto x = ctx.Input("X"); - auto label = ctx.Input("Label"); - auto dy = ctx.Input(framework::GradVarName("Y")); - PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank should be 2."); - PADDLE_ENFORCE_EQ(dy->dims().size(), 2, - "Input(Y@Grad)'s rank should be 2."); - PADDLE_ENFORCE_EQ(label->dims().size(), 2, - "Input(Label)'s rank should be 2."); - PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Y")), + "Input(Y@GRAD) shoudl be not null."); + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")), + "Output(X@GRAD) should be not null."); + + auto x_dims = ctx->GetInputDim("X"); + auto label_dims = ctx->GetInputDim("Label"); + auto dy_dims = ctx->GetInputDim(framework::GradVarName("Y")); + PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank should be 2."); + PADDLE_ENFORCE_EQ(dy_dims.size(), 2, "Input(Y@Grad)'s rank should be 2."); + PADDLE_ENFORCE_EQ(label_dims.size(), 2, "Input(Label)'s rank should be 2."); + PADDLE_ENFORCE_EQ(x_dims[0], label_dims[0], "The 1st dimension of Input(X) and Input(Label) should " "be equal."); - PADDLE_ENFORCE_EQ(x->dims()[0], dy->dims()[0], + PADDLE_ENFORCE_EQ(x_dims[0], dy_dims[0], "The 1st dimension of Input(X) and Input(Y@Grad) should " "be equal."); - PADDLE_ENFORCE_EQ(dy->dims()[1], 1, + PADDLE_ENFORCE_EQ(dy_dims[1], 1, "The 2nd dimension of Input(Y@Grad) should be 1."); - if (ctx.Attr("softLabel")) { - PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], + if (ctx->Attrs().Get("softLabel")) { + PADDLE_ENFORCE_EQ(x_dims[1], label_dims[1], "When Attr(softLabel) == true, the 2nd dimension of " "Input(X) and Input(Label) should be equal."); } else { - PADDLE_ENFORCE_EQ(label->dims()[1], 1, + PADDLE_ENFORCE_EQ(label_dims[1], 1, "When Attr(softLabel) == false, the 2nd dimension of " "Input(Label) should be 1."); } - - auto dx = ctx.Output(framework::GradVarName("X")); - dx->Resize(x->dims()); + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); } }; class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { public: - CrossEntropyOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + CrossEntropyOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor, default Tensor), a 2-D tensor with shape N x D, " diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu index 18e44d77c9f62b296dc57952e546f844670c7d57..1cfeb7a53b047541322ac53c5b7249e660039d5c 100644 --- a/paddle/operators/cross_entropy_op.cu +++ b/paddle/operators/cross_entropy_op.cu @@ -12,62 +12,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/framework/op_registry.h" #include "paddle/operators/cross_entropy_op.h" -#include "paddle/platform/assert.h" -#include "paddle/platform/hostdevice.h" namespace paddle { namespace operators { -template -__global__ void CrossEntropyKernel(T* Y, const T* X, const int* label, - const int N, const int D) { - // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. - // CUDA_1D_KERNEL_LOOP(i, N) { - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; - i += blockDim.x * gridDim.x) { - PADDLE_ASSERT(label[i] >= 0 && label[i] < D); - Y[i] = -TolerableValue()(log(X[i * D + label[i]])); - } -} - -template -__device__ __forceinline__ T sum_single_warp(T val) { - val += __shfl_down(val, 16); - val += __shfl_down(val, 8); - val += __shfl_down(val, 4); - val += __shfl_down(val, 2); - val += __shfl_down(val, 1); - return val; -} - -template -__global__ void SoftCrossEntropyKernel(T* Y, const T* X, const T* label, - const int class_num) { - int tid = threadIdx.x; - extern __shared__ T d_sum[]; - d_sum[tid] = 0; - - int cur_idx = tid; - int next_idx = blockIdx.x * class_num + tid; - while (cur_idx < class_num) { - d_sum[tid] += TolerableValue()(std::log(X[next_idx])) * label[next_idx]; - next_idx += blockDim.x; - cur_idx += blockDim.x; - } - __syncthreads(); - - for (unsigned int stride = blockDim.x >> 1; stride >= 32; stride >>= 1) { - if (tid < stride) d_sum[tid] += d_sum[tid + stride]; - __syncthreads(); - } - - T val = d_sum[tid]; - val = sum_single_warp(val); - if (tid == 0) Y[blockIdx.x] = -val; -} - +namespace { // TODO(qingqing): make zero setting a common function. template __global__ void Zero(T* X, const int N) { @@ -100,6 +50,7 @@ __global__ void SoftCrossEntropyGradientKernel(T* dX, const T* dY, const T* X, dX[ids] = -label[ids] * dY[row_ids] / X[ids]; } } +} // namespace template class CrossEntropyOpCUDAKernel : public framework::OpKernel { @@ -107,36 +58,13 @@ class CrossEntropyOpCUDAKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), "This kernel only runs on GPU device."); - const Tensor* x = ctx.Input("X"); const Tensor* label = ctx.Input("Label"); Tensor* y = ctx.Output("Y"); + y->mutable_data(ctx.GetPlace()); - const T* x_data = x->data(); - T* y_data = y->mutable_data(ctx.GetPlace()); - - int batch_size = x->dims()[0]; - int class_num = x->dims()[1]; - - if (ctx.Attr("softLabel")) { - auto* label_data = ctx.Input("Label")->data(); - int block = class_num > 512 ? 512 : pow(2, int(std::log2(class_num))); - - SoftCrossEntropyKernel< - T><<( - ctx.device_context()) - .stream()>>>(y_data, x_data, label_data, class_num); - } else { - auto* label_data = ctx.Input("Label")->data(); - int block = 512; - int grid = (batch_size + block - 1) / block; - CrossEntropyKernel<<< - grid, block, 0, reinterpret_cast( - ctx.device_context()) - .stream()>>>(y_data, x_data, label_data, - batch_size, class_num); - } + math::CrossEntropyFunctor()( + ctx, y, x, label, ctx.Attr("softLabel")); } }; @@ -150,6 +78,7 @@ class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { const Tensor* x = ctx.Input("X"); const Tensor* label = ctx.Input("Label"); Tensor* dx = ctx.Output(framework::GradVarName("X")); + dx->mutable_data(ctx.GetPlace()); const T* dy_data = ctx.Input(framework::GradVarName("Y"))->data(); diff --git a/paddle/operators/cross_entropy_op.h b/paddle/operators/cross_entropy_op.h index 255b2e9f5ea7566cca7fd3914e38da804b7c7006..1f67461d3fadb1a979832ad049d4e0098256b834 100644 --- a/paddle/operators/cross_entropy_op.h +++ b/paddle/operators/cross_entropy_op.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" -#include "paddle/platform/hostdevice.h" +#include "paddle/operators/math/cross_entropy.h" namespace paddle { namespace operators { @@ -25,18 +25,6 @@ template using EigenMatrix = framework::EigenMatrix; -template -struct TolerableValue { - HOSTDEVICE T operator()(const T& x) const { - PADDLE_ASSERT(std::is_floating_point::value); - const T kApproInf = 1e20; - - if (x == INFINITY) return kApproInf; - if (x == -INFINITY) return -kApproInf; - return x; - } -}; - template class CrossEntropyOpKernel : public framework::OpKernel { public: @@ -46,28 +34,10 @@ class CrossEntropyOpKernel : public framework::OpKernel { const Tensor* x = ctx.Input("X"); const Tensor* labels = ctx.Input("Label"); Tensor* y = ctx.Output("Y"); - T* y_data = y->mutable_data(ctx.GetPlace()); - - const int batch_size = x->dims()[0]; - if (ctx.Attr("softLabel")) { - auto prob = EigenMatrix::From(*x); - auto lbl_mat = EigenMatrix::From(*labels); - auto loss = EigenMatrix::From(*y); + y->mutable_data(ctx.GetPlace()); - loss.device(ctx.GetEigenDevice()) = - -((lbl_mat * prob.log().unaryExpr(TolerableValue())) - .sum(Eigen::DSizes(1)) - .reshape(Eigen::DSizes(batch_size, 1))); - } else { - const int class_num = x->dims()[1]; - const T* x_data = x->data(); - - const int* label_data = labels->data(); - for (int i = 0; i < batch_size; ++i) { - int index = i * class_num + label_data[i]; - y_data[i] = -TolerableValue()(std::log(x_data[index])); - } - } + math::CrossEntropyFunctor()( + ctx, y, x, labels, ctx.Attr("softLabel")); } }; diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc index 2130eda6a42c893d8ec251a7022a0bfa44433bb7..a669b5cf00f1f4ad351486e2977bf8a76aa5bf62 100644 --- a/paddle/operators/dropout_op.cc +++ b/paddle/operators/dropout_op.cc @@ -24,25 +24,25 @@ class DropoutOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); - PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); - PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); - - auto dims = ctx.Input("X")->dims(); - ctx.Output("Out")->Resize(dims); - if (ctx.Attr("is_training")) { - ctx.Output("Mask")->Resize(dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_GE(ctx->Attrs().Get("dropout_prob"), 0); + PADDLE_ENFORCE_LE(ctx->Attrs().Get("dropout_prob"), 1); + + auto x_dims = ctx->GetInputDim("X"); + ctx->SetOutputDim("Out", x_dims); + if (ctx->Attrs().Get("is_training") == 1) { + ctx->SetOutputDim("Mask", x_dims); } - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->ShareLoD("X", /*->*/ "Out"); } }; template class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { public: - DropoutOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + DropoutOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddAttr("dropout_prob", "Probability of setting units to zero.") .SetDefault(.5f); @@ -70,27 +70,26 @@ class DropoutOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE(ctx.Attr("is_training"), - "GradOp is only callable when is_training is true"); - - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Mask"), "Mask must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) must not be null."); - - PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); - PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); - auto x_dims = ctx.Input("X")->dims(); - auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE_EQ(ctx->Attrs().Get("is_training"), 1, + "GradOp is only callable when is_training is true"); + + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); + PADDLE_ENFORCE(ctx->HasInput("Mask"), "Mask must not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) must not be null."); + + PADDLE_ENFORCE_GE(ctx->Attrs().Get("dropout_prob"), 0); + PADDLE_ENFORCE_LE(ctx->Attrs().Get("dropout_prob"), 1); + auto x_dims = ctx->GetInputDim("X"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); PADDLE_ENFORCE_EQ(x_dims, out_dims, "Dimensions of Input(X) and Out@Grad must be the same."); - auto mask_dims = ctx.Input("Mask")->dims(); + auto mask_dims = ctx->GetInputDim("Mask"); PADDLE_ENFORCE_EQ(x_dims, mask_dims, "Dimensions of Input(X) and Mask must be the same."); - auto *x_grad = ctx.Output(framework::GradVarName("X")); - x_grad->Resize(x_dims); + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); } }; diff --git a/paddle/operators/elementwise_add_op.cc b/paddle/operators/elementwise_add_op.cc index 5f7b654d69f081dfa85b0d61960eb52b7982faa1..d9bc80c869c023caebf0b45ed24f2def3f0b1dd8 100644 --- a/paddle/operators/elementwise_add_op.cc +++ b/paddle/operators/elementwise_add_op.cc @@ -13,6 +13,7 @@ limitations under the License. */ #include "paddle/operators/elementwise_add_op.h" +#include "paddle/operators/elementwise_op.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h index 9e9f1ffba6fb23f5394713c67aa4363b85717f50..e9f78ef26e05878053d968c35f17b456c128827a 100644 --- a/paddle/operators/elementwise_add_op.h +++ b/paddle/operators/elementwise_add_op.h @@ -14,7 +14,7 @@ #pragma once -#include "paddle/operators/elementwise_op.h" +#include "paddle/operators/elementwise_op_function.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_div_op.cc b/paddle/operators/elementwise_div_op.cc index c6898150d310d0c4fdefae5a58a5792a72f9889e..3f56344d0007b5f14fd9b5b9b44a9b29d3c42f2a 100644 --- a/paddle/operators/elementwise_div_op.cc +++ b/paddle/operators/elementwise_div_op.cc @@ -13,6 +13,7 @@ limitations under the License. */ #include "paddle/operators/elementwise_div_op.h" +#include "paddle/operators/elementwise_op.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h index 9bd7c8ea548c46ec9b4c5a085e4e70d5dd162f3a..99b6d9c1991edfb0018f8a459dfa373948cec434 100644 --- a/paddle/operators/elementwise_div_op.h +++ b/paddle/operators/elementwise_div_op.h @@ -14,7 +14,7 @@ #pragma once -#include "paddle/operators/elementwise_op.h" +#include "paddle/operators/elementwise_op_function.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc index f2544b54d6bc543a50d8de03d482333b485bc076..bda5dfe03e974740fe4a07191ae6b68ebfcd5d3a 100644 --- a/paddle/operators/elementwise_mul_op.cc +++ b/paddle/operators/elementwise_mul_op.cc @@ -13,6 +13,7 @@ limitations under the License. */ #include "paddle/operators/elementwise_mul_op.h" +#include "paddle/operators/elementwise_op.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_mul_op.h b/paddle/operators/elementwise_mul_op.h index 1eaf2e3efc97a32739efcaf37066817ee173fadc..6ab642378bb0af8593ca0677014aede3c03cff8e 100644 --- a/paddle/operators/elementwise_mul_op.h +++ b/paddle/operators/elementwise_mul_op.h @@ -13,7 +13,7 @@ limitations under the License. */ #pragma once -#include "paddle/operators/elementwise_op.h" +#include "paddle/operators/elementwise_op_function.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h index f224722c1bec6716e68de9da2509250f7d4b37ae..3082f37422faa990bbf03c8a1a87b025d481a290 100644 --- a/paddle/operators/elementwise_op.h +++ b/paddle/operators/elementwise_op.h @@ -1,222 +1,44 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #pragma once -#include -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" -#include "paddle/operators/math/math_function.h" +#include "paddle/framework/operator.h" namespace paddle { namespace operators { -/* - * Out = X ⊙ Y - * If Y's shape does not match X' shape, they will be reshaped. - * For example: - * 1. shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 - * pre=2, n=3*4, post=5 - * x.shape(2, 12, 5) * y.shape(1,12,1).broadcast(2,12,5) - * 2. shape(X) = (2, 3, 4, 5), shape(Y) = (4,5) - * pre=2*3, n=4*5, post=1 - * x.shape(2, 3, 20) * y.shape(1,1,20).broadcast(2,3,20) - */ -inline void get_mid_dims(const framework::DDim& x_dims, - const framework::DDim& y_dims, const int axis, - int& pre, int& n, int& post) { - pre = 1; - n = 1; - post = 1; - for (int i = 0; i < axis; ++i) { - pre *= x_dims[i]; - } - - for (int i = 0; i < y_dims.size(); ++i) { - PADDLE_ENFORCE_EQ(x_dims[i + axis], y_dims[i], - "Broadcast dimension mismatch."); - n *= y_dims[i]; - } - - for (int i = axis + y_dims.size(); i < x_dims.size(); ++i) { - post *= x_dims[i]; - } -} - -#define EIGEN_FUNCTOR(name, eigen_op) \ - struct Eigen##name##Functor { \ - template \ - inline void Run(const framework::Tensor* x, const framework::Tensor* y, \ - framework::Tensor* z, \ - const framework::ExecutionContext& ctx) { \ - auto x_e = framework::EigenVector::Flatten(*x); \ - auto y_e = framework::EigenVector::Flatten(*y); \ - auto z_e = framework::EigenVector::Flatten(*z); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_e); \ - } \ - template \ - inline void RunBroadCast(const framework::Tensor* x, \ - const framework::Tensor* y, framework::Tensor* z, \ - const framework::ExecutionContext& ctx, int pre, \ - int n) { \ - auto x_e = framework::EigenVector::Flatten(*x); \ - auto y_e = framework::EigenVector::Flatten(*y); \ - auto z_e = framework::EigenVector::Flatten(*z); \ - auto y_bcast = y_e.reshape(Eigen::DSizes(1, n)) \ - .broadcast(Eigen::DSizes(pre, 1)) \ - .reshape(Eigen::DSizes(x_e.size())); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ - } \ - template \ - inline void RunBroadCast2(const framework::Tensor* x, \ - const framework::Tensor* y, \ - framework::Tensor* z, \ - const framework::ExecutionContext& ctx, int pre, \ - int n, int post) { \ - auto x_e = framework::EigenVector::Flatten(*x); \ - auto y_e = framework::EigenVector::Flatten(*y); \ - auto z_e = framework::EigenVector::Flatten(*z); \ - auto y_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) \ - .broadcast(Eigen::DSizes(pre, 1, post)) \ - .reshape(Eigen::DSizes(x_e.size())); \ - z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ - } \ - } - -template -void ElementwiseCompute(const framework::ExecutionContext& ctx) { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input.") - - if (x_dims == y_dims || product(y_dims) == 1) { - functor f; - f.template Run(x, y, z, ctx); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - functor f; - f.template RunBroadCast(x, y, z, ctx, pre, n); - return; - } else { - functor f; - f.template RunBroadCast2(x, y, z, ctx, pre, n, post); - return; - } -} - -#define EIGEN_ADD(x, y) ((x) + (y)) -EIGEN_FUNCTOR(Add, EIGEN_ADD); - -#define EIGEN_SUB(x, y) ((x) - (y)) -EIGEN_FUNCTOR(Sub, EIGEN_SUB); - -#define EIGEN_MUL(x, y) ((x) * (y)) -EIGEN_FUNCTOR(Mul, EIGEN_MUL); - -#define EIGEN_DIV(x, y) ((x) / (y)) -EIGEN_FUNCTOR(Div, EIGEN_DIV); - -template -void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* out = ctx.Input("Out"); - auto* dout = ctx.Input(framework::GradVarName("Out")); - - auto place = ctx.GetEigenDevice(); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - - auto* dx = ctx.Output(framework::GradVarName("X")); - auto* dy = ctx.Output(framework::GradVarName("Y")); - if (dx) { - dx->mutable_data(ctx.GetPlace()); - } - if (dy) { - dy->mutable_data(ctx.GetPlace()); - } - - if (x_dims == y_dims) { - functor f; - f(place, x, y, out, dx, dy, dout); - return; - } - - if (product(y_dims) == 1) { - functor1 f; - f(place, x, y, out, dx, dy, dout); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - - if (post == 1) { - broadcastfunctor f; - f(place, x, y, out, dx, dy, dout, pre, n); - return; - } else { - broadcast2functor f; - f(place, x, y, out, dx, dy, dout, pre, n, post); - return; - } -} - class ElementwiseOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; protected: using Tensor = framework::Tensor; - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of elementwise op should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of elementwise op should not be null"); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of elementwise op should not be null."); - - auto x_dim = ctx.Input("X")->dims(); - auto y_dim = ctx.Input("Y")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of elementwise op should not be null"); + PADDLE_ENFORCE(ctx->HasInput("Y"), + "Input(Y) of elementwise op should not be null"); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of elementwise op should not be null."); + + auto x_dim = ctx->GetInputDim("X"); + auto y_dim = ctx->GetInputDim("Y"); PADDLE_ENFORCE_GE(x_dim.size(), y_dim.size(), "Rank of first input must >= rank of second input.") - ctx.Output("Out")->Resize(x_dim); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", x_dim); + ctx->ShareLoD("X", /*->*/ "Out"); } }; @@ -234,7 +56,7 @@ must be small or equal to X's dimensions. )DOC"); AddAttr("axis", R"DOC( -When the shape(Y) does not equal the shape(X),Y will be broadcasted +When the shape(Y) does not equal the shape(X),Y will be broadcasted to match the shape of X and axis should be dimension index Y in X )DOC") .SetDefault(-1) @@ -244,7 +66,7 @@ to match the shape of X and axis should be dimension index Y in X comment_ = R"DOC( Limited elementwise {name} operator.The equation is: Out = {equation}. 1. The shape of Y should be same with X or -2. Y's shape is a subset of X. +2. Y's shape is a subset of X. Y will be broadcasted to match the shape of X and axis should be dimension index Y in X. example: @@ -284,27 +106,26 @@ class ElementwiseOpGrad : public framework::OperatorWithKernel { using Tensor = framework::Tensor; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - auto* x_grad = ctx.Output(framework::GradVarName("X")); - auto* y_grad = ctx.Output(framework::GradVarName("Y")); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), "Rank of first input must >= rank of second input.") - if (x_grad) { - x_grad->Resize(x_dims); + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); } - - if (y_grad) { - y_grad->Resize(y_dims); + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); } } }; diff --git a/paddle/operators/elementwise_op_function.h b/paddle/operators/elementwise_op_function.h new file mode 100644 index 0000000000000000000000000000000000000000..3eb97f60b59848d23bcd15ea1e3d2f21b721f6a4 --- /dev/null +++ b/paddle/operators/elementwise_op_function.h @@ -0,0 +1,200 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +/* + * Out = X ⊙ Y + * If Y's shape does not match X' shape, they will be reshaped. + * For example: + * 1. shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 + * pre=2, n=3*4, post=5 + * x.shape(2, 12, 5) * y.shape(1,12,1).broadcast(2,12,5) + * 2. shape(X) = (2, 3, 4, 5), shape(Y) = (4,5) + * pre=2*3, n=4*5, post=1 + * x.shape(2, 3, 20) * y.shape(1,1,20).broadcast(2,3,20) + */ +inline void get_mid_dims(const framework::DDim& x_dims, + const framework::DDim& y_dims, const int axis, + int& pre, int& n, int& post) { + pre = 1; + n = 1; + post = 1; + for (int i = 0; i < axis; ++i) { + pre *= x_dims[i]; + } + + for (int i = 0; i < y_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(x_dims[i + axis], y_dims[i], + "Broadcast dimension mismatch."); + n *= y_dims[i]; + } + + for (int i = axis + y_dims.size(); i < x_dims.size(); ++i) { + post *= x_dims[i]; + } +} + +#define EIGEN_FUNCTOR(name, eigen_op) \ + struct Eigen##name##Functor { \ + template \ + inline void Run(const framework::Tensor* x, const framework::Tensor* y, \ + framework::Tensor* z, \ + const framework::ExecutionContext& ctx) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_e); \ + } \ + template \ + inline void RunBroadCast(const framework::Tensor* x, \ + const framework::Tensor* y, framework::Tensor* z, \ + const framework::ExecutionContext& ctx, int pre, \ + int n) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + auto y_bcast = y_e.reshape(Eigen::DSizes(1, n)) \ + .broadcast(Eigen::DSizes(pre, 1)) \ + .reshape(Eigen::DSizes(x_e.size())); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + } \ + template \ + inline void RunBroadCast2(const framework::Tensor* x, \ + const framework::Tensor* y, \ + framework::Tensor* z, \ + const framework::ExecutionContext& ctx, int pre, \ + int n, int post) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + auto y_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) \ + .broadcast(Eigen::DSizes(pre, 1, post)) \ + .reshape(Eigen::DSizes(x_e.size())); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + } \ + } + +template +void ElementwiseCompute(const framework::ExecutionContext& ctx) { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* z = ctx.Output("Out"); + z->mutable_data(ctx.GetPlace()); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input.") + + if (x_dims == y_dims || product(y_dims) == 1) { + functor f; + f.template Run(x, y, z, ctx); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), + "Axis should be in range [0, x_dims)"); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + if (post == 1) { + functor f; + f.template RunBroadCast(x, y, z, ctx, pre, n); + return; + } else { + functor f; + f.template RunBroadCast2(x, y, z, ctx, pre, n, post); + return; + } +} + +#define EIGEN_ADD(x, y) ((x) + (y)) +EIGEN_FUNCTOR(Add, EIGEN_ADD); + +#define EIGEN_SUB(x, y) ((x) - (y)) +EIGEN_FUNCTOR(Sub, EIGEN_SUB); + +#define EIGEN_MUL(x, y) ((x) * (y)) +EIGEN_FUNCTOR(Mul, EIGEN_MUL); + +#define EIGEN_DIV(x, y) ((x) / (y)) +EIGEN_FUNCTOR(Div, EIGEN_DIV); + +template +void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* out = ctx.Input("Out"); + auto* dout = ctx.Input(framework::GradVarName("Out")); + + auto place = ctx.GetEigenDevice(); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + + auto* dx = ctx.Output(framework::GradVarName("X")); + auto* dy = ctx.Output(framework::GradVarName("Y")); + if (dx) { + dx->mutable_data(ctx.GetPlace()); + } + if (dy) { + dy->mutable_data(ctx.GetPlace()); + } + + if (x_dims == y_dims) { + functor f; + f(place, x, y, out, dx, dy, dout); + return; + } + + if (product(y_dims) == 1) { + functor1 f; + f(place, x, y, out, dx, dy, dout); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + + if (post == 1) { + broadcastfunctor f; + f(place, x, y, out, dx, dy, dout, pre, n); + return; + } else { + broadcast2functor f; + f(place, x, y, out, dx, dy, dout, pre, n, post); + return; + } +} +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/elementwise_sub_op.cc b/paddle/operators/elementwise_sub_op.cc index 31c37ff7ab5595c29f973929387d3945b6f3aaf8..3e4f98fdb35b148931a67d511fe41958eb523f99 100644 --- a/paddle/operators/elementwise_sub_op.cc +++ b/paddle/operators/elementwise_sub_op.cc @@ -13,6 +13,7 @@ limitations under the License. */ #include "paddle/operators/elementwise_sub_op.h" +#include "paddle/operators/elementwise_op.h" namespace paddle { namespace operators { diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h index f6bc66cd0e1594a8bc7070e2f182401b92d1c88e..3ca1376c73b3332b76a5973e201f9e4fba77cd21 100644 --- a/paddle/operators/elementwise_sub_op.h +++ b/paddle/operators/elementwise_sub_op.h @@ -13,7 +13,7 @@ limitations under the License. */ #pragma once -#include "paddle/operators/elementwise_op.h" +#include "paddle/operators/elementwise_op_function.h" namespace paddle { namespace operators { diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index 761a527a5574edc779340ec595dfe1bc1964438a..e164de6584e7350283781019cc74118c2d13646e 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -22,15 +22,13 @@ class FillZerosLikeOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of FillZerosLikeOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) of FillZerosLikeOp should not be null."); - - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); - ctx.ShareLoD("X", /*->*/ "Y"); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of FillZerosLikeOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Y"), + "Output(Y) of FillZerosLikeOp should not be null."); + ctx->SetOutputDim("Y", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Y"); } }; diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc index fecd1ce2147a1e6f2f7928266be74ed7b647c5b9..0e3cd174adee1e50d0a63861286a26d325484efb 100644 --- a/paddle/operators/gather_op.cc +++ b/paddle/operators/gather_op.cc @@ -23,19 +23,19 @@ class GatherOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of GatherOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Index"), - "Input(Index) of GatherOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of GatherOp should not be null."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of GatherOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Index"), + "Input(Index) of GatherOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of GatherOp should not be null."); - int batch_size = ctx.Input("Index")->dims()[0]; + int batch_size = ctx->GetInputDim("Index")[0]; PADDLE_ENFORCE_GE(batch_size, 0, "Batch size must be >0"); - framework::DDim output_dims(ctx.Input("X")->dims()); + framework::DDim output_dims(ctx->GetInputDim("X")); output_dims[0] = batch_size; - ctx.Output("Out")->Resize(output_dims); + ctx->SetOutputDim("Out", output_dims); } }; @@ -44,23 +44,20 @@ class GatherGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - auto X_grad = ctx.Output(framework::GradVarName("X")); - auto X = ctx.Input("X"); - - X_grad->Resize(X->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } }; class GatherOpMaker : public framework::OpProtoAndCheckerMaker { public: - GatherOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + GatherOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The source input of gather op"); AddInput("Index", "The index input of gather op"); AddOutput("Out", "The output of add op"); AddComment(R"DOC( -Gather Operator by selecting from the first axis, +Gather Operator by selecting from the first axis, Out = X[Index] )DOC"); diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index 5b7cbb5cc7bcb7e43b15363d37d7b8f2cbf0fbdc..05120a6e7bcfdb8641c722731f462c89e4223339 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -43,13 +43,10 @@ class GaussianRandomOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of GaussianRandomOp should not be null."); - - auto* tensor = ctx.Output("Out"); - auto dims = Attr>("dims"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of GaussianRandomOp should not be null."); + auto dims = ctx->Attrs().Get>("dims"); std::vector temp; temp.reserve(dims.size()); for (auto dim : dims) { @@ -57,7 +54,7 @@ class GaussianRandomOp : public framework::OperatorWithKernel { } PADDLE_ENFORCE(dims.size() > 0UL, "dims can be one int or array. dims must be set."); - tensor->Resize(framework::make_ddim(temp)); + ctx->SetOutputDim("Out", framework::make_ddim(temp)); } }; diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc index 04ac24662e9cfec6a49cd213cb76bdebc7b730c8..9b1314bfbade8551d98b0fbabb7c2968d7600db5 100644 --- a/paddle/operators/lookup_table_op.cc +++ b/paddle/operators/lookup_table_op.cc @@ -22,27 +22,26 @@ class LookupTableOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("W"), - "Input(W) of LookupTableOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Ids"), - "Input(Ids) of LookupTableOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of LookupTableOp should not be null."); - - auto table_t = ctx.Input("W"); - auto ids_t = ctx.Input("Ids"); - auto output_t = ctx.Output("Out"); - - output_t->Resize({ids_t->dims()[0], table_t->dims()[1]}); - ctx.ShareLoD("Ids", /*->*/ "Out"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("W"), + "Input(W) of LookupTableOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Ids"), + "Input(Ids) of LookupTableOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of LookupTableOp should not be null."); + + auto table_dims = ctx->GetInputDim("W"); + auto ids_dims = ctx->GetInputDim("Ids"); + + ctx->SetOutputDim("Out", {ids_dims[0], table_dims[1]}); + ctx->ShareLoD("Ids", /*->*/ "Out"); } }; class LookupTableOpMaker : public framework::OpProtoAndCheckerMaker { public: - LookupTableOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + LookupTableOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("W", "An input represents embedding tensors," @@ -66,11 +65,9 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &context) const override { - auto table = context.Input("W"); - auto d_table = - context.Output(framework::GradVarName("W")); - d_table->Resize(table->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + auto table_dims = ctx->GetInputDim("W"); + ctx->SetOutputDim(framework::GradVarName("W"), table_dims); } }; diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc index 3600f199770c4b8c9a6561b4c270a91bc8b20c0b..bd75b001cb87d914f6c56ea35dcb5013d68145b2 100644 --- a/paddle/operators/lstm_unit_op.cc +++ b/paddle/operators/lstm_unit_op.cc @@ -22,37 +22,36 @@ class LstmUnitOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of LSTM should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("C_prev"), - "Input(C_prev) of LSTM should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("C"), - "Output(C) of LSTM should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("H"), - "Output(H) of LSTM should not be null."); - - auto *x = ctx.Input("X"); - auto *c_prev = ctx.Input("C_prev"); - - PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank must be 2."); - PADDLE_ENFORCE(x->dims()[0] == c_prev->dims()[0], + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("C_prev"), + "Input(C_prev) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("C"), + "Output(C) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("H"), + "Output(H) of LSTM should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + auto c_prev_dims = ctx->GetInputDim("C_prev"); + + PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2."); + PADDLE_ENFORCE(x_dims[0] == c_prev_dims[0], "Batch size of inputs and states must be equal"); - PADDLE_ENFORCE(x->dims()[1] == c_prev->dims()[1] * 4, + PADDLE_ENFORCE(x_dims[1] == c_prev_dims[1] * 4, "Dimension of FC should equal to prev state * 4"); - int b_size = c_prev->dims()[0]; // batch size - int s_dim = c_prev->dims()[1]; // state dim - ctx.Output("C")->Resize({b_size, s_dim}); - ctx.Output("H")->Resize({b_size, s_dim}); + int b_size = c_prev_dims[0]; // batch size + int s_dim = c_prev_dims[1]; // state dim + ctx->SetOutputDim("C", {b_size, s_dim}); + ctx->SetOutputDim("H", {b_size, s_dim}); } }; template class LstmUnitOpMaker : public framework::OpProtoAndCheckerMaker { public: - LstmUnitOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + LstmUnitOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "FC input before the non-linear activation."); AddInput( @@ -63,11 +62,11 @@ class LstmUnitOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC(Lstm-Unit Operator -Equation: +Equation: i, f, o, j = split(X) C = C_prev * sigm(f + forget_bias) + sigm(i) * tanh(j) H = C * sigm(o) - + )DOC"); AddAttr("forget_bias", "The forget bias of Lstm Unit.") .SetDefault(0.0); @@ -79,15 +78,14 @@ class LstmUnitGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("C")), - "Input(C@GRAD) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("H")), - "Input(H@GRAD) should not be null"); - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("X")->dims()); - ctx.Output(framework::GradVarName("C_prev")) - ->Resize(ctx.Input("C_prev")->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("C")), + "Input(C@GRAD) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("H")), + "Input(H@GRAD) should not be null"); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->SetOutputDim(framework::GradVarName("C_prev"), + ctx->GetInputDim("C_prev")); } }; diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index f8333f34f7b4c7b0f9a0f14a7a33f9d98e1d331c..91ae3d49f1df51d9524547f7765285bff9dbb5c5 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -1,9 +1,15 @@ - if(WITH_GPU) - nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc - im2col.cu DEPS cblas device_context) + nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc + im2col.cu DEPS cblas device_context operator) + nv_library(softmax_function SRCS softmax.cc softmax.cu + DEPS operator) + nv_library(cross_entropy_function SRCS cross_entropy.cc cross_entropy.cu + DEPS operator) else() - cc_library(math_function SRCS math_function.cc im2col.cc DEPS cblas device_context) + cc_library(math_function SRCS math_function.cc im2col.cc + DEPS cblas device_context operator) + cc_library(softmax_function SRCS softmax.cc DEPS operator) + cc_library(cross_entropy_function SRCS cross_entropy.cc DEPS operator) endif() nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) diff --git a/paddle/operators/math/cross_entropy.cc b/paddle/operators/math/cross_entropy.cc new file mode 100644 index 0000000000000000000000000000000000000000..a5a426bc7b16852e67afd790df7a91d89a458c8a --- /dev/null +++ b/paddle/operators/math/cross_entropy.cc @@ -0,0 +1,59 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/math/cross_entropy.h" + +namespace paddle { +namespace operators { +namespace math { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +template +class CrossEntropyFunctor { + public: + void operator()(const framework::ExecutionContext& ctx, + framework::Tensor* out, const framework::Tensor* prob, + const framework::Tensor* labels, const bool softLabel) { + const int batch_size = prob->dims()[0]; + if (softLabel) { + auto in = EigenMatrix::From(*prob); + auto lbl = EigenMatrix::From(*labels); + auto loss = EigenMatrix::From(*out); + + loss.device(ctx.GetEigenDevice()) = + -((lbl * in.log().unaryExpr(math::TolerableValue())) + .sum(Eigen::DSizes(1)) + .reshape(Eigen::DSizes(batch_size, 1))); + } else { + const int class_num = prob->dims()[1]; + const T* prob_data = prob->data(); + T* loss_data = out->data(); + + const int* label_data = labels->data(); + for (int i = 0; i < batch_size; ++i) { + int index = i * class_num + label_data[i]; + loss_data[i] = -math::TolerableValue()(std::log(prob_data[index])); + } + } + } +}; + +template class CrossEntropyFunctor; +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/cross_entropy.cu b/paddle/operators/math/cross_entropy.cu new file mode 100644 index 0000000000000000000000000000000000000000..d14a75a30c01deb86937a3ced43005aed4066d86 --- /dev/null +++ b/paddle/operators/math/cross_entropy.cu @@ -0,0 +1,111 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/math/cross_entropy.h" + +namespace paddle { +namespace operators { +namespace math { + +namespace { +template +__global__ void CrossEntropyKernel(T* Y, const T* X, const int* label, + const int N, const int D) { + // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. + // CUDA_1D_KERNEL_LOOP(i, N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + PADDLE_ASSERT(label[i] >= 0 && label[i] < D); + Y[i] = -math::TolerableValue()(log(X[i * D + label[i]])); + } +} + +template +__device__ __forceinline__ T sum_single_warp(T val) { + val += __shfl_down(val, 16); + val += __shfl_down(val, 8); + val += __shfl_down(val, 4); + val += __shfl_down(val, 2); + val += __shfl_down(val, 1); + return val; +} + +template +__global__ void SoftCrossEntropyKernel(T* Y, const T* X, const T* label, + const int class_num) { + int tid = threadIdx.x; + extern __shared__ T d_sum[]; + d_sum[tid] = 0; + + int cur_idx = tid; + int next_idx = blockIdx.x * class_num + tid; + while (cur_idx < class_num) { + d_sum[tid] += + math::TolerableValue()(std::log(X[next_idx])) * label[next_idx]; + next_idx += blockDim.x; + cur_idx += blockDim.x; + } + __syncthreads(); + + for (unsigned int stride = blockDim.x >> 1; stride >= 32; stride >>= 1) { + if (tid < stride) d_sum[tid] += d_sum[tid + stride]; + __syncthreads(); + } + + T val = d_sum[tid]; + val = sum_single_warp(val); + if (tid == 0) Y[blockIdx.x] = -val; +} +} // namespace + +using Tensor = framework::Tensor; + +template +class CrossEntropyFunctor { + public: + void operator()(const framework::ExecutionContext& ctx, + framework::Tensor* out, const framework::Tensor* prob, + const framework::Tensor* labels, bool softLabel) { + const T* prob_data = prob->data(); + T* loss_data = out->mutable_data(ctx.GetPlace()); + + int batch_size = prob->dims()[0]; + int class_num = prob->dims()[1]; + + if (softLabel) { + const T* label_data = labels->data(); + int block = class_num > 512 ? 512 : pow(2, int(std::log2(class_num))); + + SoftCrossEntropyKernel< + T><<( + ctx.device_context()) + .stream()>>>(loss_data, prob_data, label_data, class_num); + } else { + const int* label_data = labels->data(); + int block = 512; + int grid = (batch_size + block - 1) / block; + CrossEntropyKernel<<< + grid, block, 0, reinterpret_cast( + ctx.device_context()) + .stream()>>>(loss_data, prob_data, label_data, + batch_size, class_num); + } + } +}; + +template class CrossEntropyFunctor; +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/cross_entropy.h b/paddle/operators/math/cross_entropy.h new file mode 100644 index 0000000000000000000000000000000000000000..18e637cf9186b5dc21e94f1ab15b3d858ec93c67 --- /dev/null +++ b/paddle/operators/math/cross_entropy.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/operator.h" +#include "paddle/framework/tensor.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { +namespace math { + +template +struct TolerableValue { + HOSTDEVICE T operator()(const T& x) const { + PADDLE_ASSERT(std::is_floating_point::value); + const T kApproInf = 1e20; + + if (x == INFINITY) return kApproInf; + if (x == -INFINITY) return -kApproInf; + return x; + } +}; + +template +class CrossEntropyFunctor { + public: + // (TODO caoying) it is much better to use DeviceContext as the first + // parameter. + void operator()(const framework::ExecutionContext& context, + framework::Tensor* out, const framework::Tensor* prob, + const framework::Tensor* labels, const bool softLabel); +}; +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/softmax.cc b/paddle/operators/math/softmax.cc new file mode 100644 index 0000000000000000000000000000000000000000..ac9f3c4bf61bf8e13faa17387f1112756db9a100 --- /dev/null +++ b/paddle/operators/math/softmax.cc @@ -0,0 +1,25 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/math/softmax.h" + +namespace paddle { +namespace operators { +namespace math { + +template class SoftmaxFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/softmax.cu b/paddle/operators/math/softmax.cu new file mode 100644 index 0000000000000000000000000000000000000000..4c3df0550e7ca6f4310db1d35cc34d5c73a2dd16 --- /dev/null +++ b/paddle/operators/math/softmax.cu @@ -0,0 +1,27 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU + +#include "paddle/operators/math/softmax.h" + +namespace paddle { +namespace operators { +namespace math { + +template class SoftmaxFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/softmax.h b/paddle/operators/math/softmax.h new file mode 100644 index 0000000000000000000000000000000000000000..3d2f0d0aecffcd0fe51166c3d863aa8b91bba196 --- /dev/null +++ b/paddle/operators/math/softmax.h @@ -0,0 +1,73 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/operator.h" +#include "paddle/framework/tensor.h" + +namespace paddle { +namespace operators { +namespace math { + +template +using EigenMatrix = framework::EigenMatrix; + +template +struct ValueClip { + HOSTDEVICE T operator()(const T& x) const { + const T kThreshold = -64.; + return x < kThreshold ? kThreshold : x; + } +}; + +template +class SoftmaxFunctor { + public: + void operator()(const framework::ExecutionContext& context, + const framework::Tensor* X, framework::Tensor* Y) { + auto logits = EigenMatrix::From(*X); + auto softmax = EigenMatrix::From(*Y); + + const int kBatchDim = 0; + const int kClassDim = 1; + + const int batch_size = logits.dimension(kBatchDim); + const int num_classes = logits.dimension(kClassDim); + + Eigen::DSizes along_class(kClassDim); + Eigen::DSizes batch_by_one(batch_size, 1); + Eigen::DSizes one_by_class(1, num_classes); + + auto shifted_logits = (logits - + logits.maximum(along_class) + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)) + .unaryExpr(ValueClip()); + + softmax.device(context.GetEigenDevice()) = shifted_logits.exp(); + softmax.device(context.GetEigenDevice()) = + (softmax * + softmax.sum(along_class) + .inverse() + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)); + } +}; +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index b04384bda81b93f5db0be3206eee10ad5e854540..d799239d4ed6d230578c77921a1a454b476b63fa 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -22,18 +22,18 @@ class MeanOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of MeanOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of MeanOp should not be null."); - ctx.Output("Out")->Resize({1}); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of MeanOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of MeanOp should not be null."); + ctx->SetOutputDim("Out", {1}); } }; class MeanOpMaker : public framework::OpProtoAndCheckerMaker { public: - MeanOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + MeanOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of mean op"); AddOutput("Out", "The output of mean op").NotInGradient(); @@ -47,9 +47,8 @@ class MeanGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("X")->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index 29cb85489bd05f6c1e7143d962eac0af26e75825..ce049d4d7bd96a6758d71b381e6e6b4edbcc8b5c 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -26,22 +26,22 @@ class MinusOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of MinusOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of MinusOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of MinusOp should not be null."); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of MinusOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), + "Input(Y) of MinusOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of MinusOp should not be null."); - auto *left_tensor = ctx.Input("X"); - auto *right_tensor = ctx.Input("Y"); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); PADDLE_ENFORCE_EQ( - left_tensor->numel(), right_tensor->numel(), + x_dims, y_dims, "Minus operator must take two tensor with same num of elements"); - ctx.Output("Out")->Resize(left_tensor->dims()); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", /*->*/ "Out"); } }; diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index 8606c0d1e1bf7a52299528d30af0367d9f93edd2..84212a2b3be1ac3664ebd77c7a0ae4d86abad3a0 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -22,20 +22,19 @@ class ModifiedHuberLossOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& context) const override { - PADDLE_ENFORCE_NOT_NULL(context.InputVar("X"), "X must be initialized."); - PADDLE_ENFORCE_NOT_NULL(context.InputVar("Y"), "Y must be initialized."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized."); - auto* x = context.Input("X"); - auto* y = context.Input("Y"); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); - PADDLE_ENFORCE_EQ(x->dims(), y->dims(), - "The shape of X and Y must be the same."); - PADDLE_ENFORCE_EQ(x->dims().size(), 2, "The tensor rank of X must be 2."); - PADDLE_ENFORCE_EQ(x->dims()[1], 1, "The 2nd dimension of X must be 1."); + PADDLE_ENFORCE_EQ(x_dims, y_dims, "The shape of X and Y must be the same."); + PADDLE_ENFORCE_EQ(x_dims.size(), 2, "The tensor rank of X must be 2."); + PADDLE_ENFORCE_EQ(x_dims[1], 1, "The 2nd dimension of X must be 1."); - context.Output("IntermediateVal")->Resize(x->dims()); - context.Output("Out")->Resize({x->dims()[0], 1}); + ctx->SetOutputDim("IntermediateVal", x_dims); + ctx->SetOutputDim("Out", {x_dims[0], 1}); } }; @@ -75,27 +74,28 @@ class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& context) const override { - auto* x = context.Input("X"); - auto* y = context.Input("Y"); - auto* intermediate_val = context.Input("IntermediateVal"); - auto* out_grad = context.Input(framework::GradVarName("Out")); - auto* x_grad = - context.Output(framework::GradVarName("X")); - - PADDLE_ENFORCE_NOT_NULL(x, "X must be initialized."); - PADDLE_ENFORCE_NOT_NULL(y, "Y must be initialized."); - PADDLE_ENFORCE_NOT_NULL(intermediate_val, - "Intermediate value must not be null."); - PADDLE_ENFORCE_NOT_NULL(out_grad, "Input(Out@Grad) must not be null."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("IntermediateVal"), + "Intermediate value must not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@Grad) must not be null."); + + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto intermediate_dims = ctx->GetInputDim("IntermediateVal"); + auto out_grad_dims = ctx->GetInputDim(framework::GradVarName("Out")); PADDLE_ENFORCE_EQ( - intermediate_val->dims(), x->dims(), + intermediate_dims, x_dims, "The shape of X and intermediate value must be the same."); - PADDLE_ENFORCE_EQ(out_grad->dims(), x->dims(), + PADDLE_ENFORCE_EQ(out_grad_dims, x_dims, "The shape of Input(Out@Grad) and X must be the same."); - if (x_grad) x_grad->Resize(x->dims()); + if (ctx->HasOutput(framework::GradVarName("X"))) { + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); + } } }; diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 7047718a3f1bf7e9598952efa1d9bcb20d5cf5b4..9858c4d9c2195c7bd0e767aaa86a950e0a791443 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -24,27 +24,23 @@ class MulOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of MulOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of MulOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of MulOp should not be null."); - - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - int x_num_col_dims = Attr("x_num_col_dims"); - int y_num_col_dims = Attr("y_num_col_dims"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of MulOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) of MulOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of MulOp should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + int x_num_col_dims = ctx->Attrs().Get("x_num_col_dims"); + int y_num_col_dims = ctx->Attrs().Get("y_num_col_dims"); PADDLE_ENFORCE(x_dims.size() > x_num_col_dims, - "The rank of input tensor X(%s) should be larger than " - "`mul_op`'s `x_num_col_dims`.", - ctx.op().Input("X")); + "The rank of input tensor X should be larger than " + "`mul_op`'s `x_num_col_dims`."); PADDLE_ENFORCE(y_dims.size() > y_num_col_dims, - "The rank of input tensor Y(%s) should be larger than " - "`mul_op`'s `y_num_col_dims`.", - ctx.op().Input("Y")); + "The rank of input tensor Y should be larger than " + "`mul_op`'s `y_num_col_dims`."); auto x_mat_dims = framework::flatten_to_2d(x_dims, x_num_col_dims); auto y_mat_dims = framework::flatten_to_2d(y_dims, y_num_col_dims); @@ -52,24 +48,23 @@ class MulOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( x_mat_dims[1], y_mat_dims[0], "First matrix's width must be equal with second matrix's height."); - ctx.Output("Out")->Resize( - {x_mat_dims[0], y_mat_dims[1]}); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", {x_mat_dims[0], y_mat_dims[1]}); + ctx->ShareLoD("X", /*->*/ "Out"); } }; class MulOpMaker : public framework::OpProtoAndCheckerMaker { public: - MulOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + MulOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The first input of mul op"); AddInput("Y", "The second input of mul op"); AddOutput("Out", "The output of mul op"); AddAttr( "x_num_col_dims", - R"DOC(mul_op can take tensors with more than two dimensions as input `X`, - in that case, tensors will be reshaped to a matrix. The matrix's first - dimension(column length) will be the product of tensor's last + R"DOC(mul_op can take tensors with more than two dimensions as input `X`, + in that case, tensors will be reshaped to a matrix. The matrix's first + dimension(column length) will be the product of tensor's last `num_col_dims` dimensions, and the matrix's second dimension(row length) will be the product of tensor's first `rank - num_col_dims` dimensions. )DOC") @@ -100,16 +95,14 @@ class MulOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - auto *x_grad = ctx.Output(framework::GradVarName("X")); - auto *y_grad = ctx.Output(framework::GradVarName("Y")); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); auto x_mat_dims = framework::flatten_to_2d(x_dims, Attr("x_num_col_dims")); @@ -125,8 +118,15 @@ class MulOpGrad : public framework::OperatorWithKernel { "The second dimension of Out@GRAD must equal to the second " "dimension of the second operand."); - if (x_grad) x_grad->Resize(x_dims); - if (y_grad) y_grad->Resize(y_dims); + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); + } } }; diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc index 7b50444d16dc57fd14b918d1159e3e21ecd1f1c4..9896d269ccc86d8fdc3bf6375e44ef5bf3e6b9c7 100644 --- a/paddle/operators/multiplex_op.cc +++ b/paddle/operators/multiplex_op.cc @@ -24,41 +24,38 @@ class MultiplexOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Ids"), - "Input(Ids) shouldn't be null."); - PADDLE_ENFORCE(!ctx.MultiInputVar("X").empty(), + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Ids"), "Input(Ids) shouldn't be null."); + PADDLE_ENFORCE(!ctx->Inputs("X").empty(), "MultiInput(X) shouldn't be empty."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) shouldn't be null."); - auto ids_dim = ctx.Input("Ids")->dims(); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) shouldn't be null."); + auto ids_dim = ctx->GetInputDim("Ids"); PADDLE_ENFORCE( ids_dim.size() == 2 && ids_dim[1] == 1, "The index tensor must be a vector with size batchSize x 1."); - auto ins = ctx.MultiInput("X"); - auto *out = ctx.Output("Out"); - auto num_ins = ins.size(); + auto ins_dims = ctx->GetInputsDim("X"); + auto num_ins = ins_dims.size(); PADDLE_ENFORCE(num_ins > 1, "multiplex operator should have more than " "one candidate input tensors."); - auto in_dim = ins[0]->dims(); + auto in_dim = ins_dims[0]; PADDLE_ENFORCE(in_dim.size() >= 2, "The rank of candidate tensors must be not less than 2."); for (size_t i = 1; i < num_ins; i++) { - auto dim = ins[i]->dims(); + auto dim = ins_dims[i]; PADDLE_ENFORCE(in_dim == dim, "All the candidate tensors must have the same size."); } - out->Resize(in_dim); + ctx->SetOutputDim("Out", in_dim); } }; class MultiplexOpMaker : public framework::OpProtoAndCheckerMaker { public: - MultiplexOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + MultiplexOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Ids", "The index tensor of multiplex operator."); AddInput("X", "The candidate tensors of multiplex operator.") @@ -88,21 +85,19 @@ class MultiplexGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE(!ctx.MultiInputVar("X").empty(), - "Input(X) should not be null."); - PADDLE_ENFORCE(!ctx.MultiOutputVar(framework::GradVarName("X")).empty(), + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(!ctx->Inputs("X").empty(), "Input(X) should not be null."); + PADDLE_ENFORCE(!ctx->Outputs(framework::GradVarName("X")).empty(), "Output(X@Grad) should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null."); - auto d_ins = ctx.MultiOutput(framework::GradVarName("X")); - auto ins = ctx.MultiInput("X"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null."); + std::vector d_ins; + auto ins = ctx->GetInputsDim("X"); // No need to compute gradient for Input(Ids) for (size_t i = 0; i < ins.size(); i++) { - if (d_ins[i]) { - d_ins[i]->Resize(ins[i]->dims()); - } + d_ins.push_back(ins[i]); } + ctx->SetOutputsDim(framework::GradVarName("X"), d_ins); } }; diff --git a/paddle/operators/multiplex_op.cu b/paddle/operators/multiplex_op.cu index 70e46815fc9148a2530d437d20c14f5d40baa1a4..505776612e7119e568493506b113661a839e5bd1 100644 --- a/paddle/operators/multiplex_op.cu +++ b/paddle/operators/multiplex_op.cu @@ -42,7 +42,7 @@ class MultiplexGPUKernel : public framework::OpKernel { for (auto i = 0; i < rows; i++) { int32_t k = index[i]; PADDLE_ENFORCE_GE(k, 0, "index must be nonnegative."); - PADDLE_ENFORCE_LT(k, ins.size(), + PADDLE_ENFORCE_LT((size_t)k, ins.size(), "index exceeds the number of candidate tensors."); memory::Copy(place, out->data() + i * cols, place, ins[k]->data() + i * cols, cols * sizeof(T), stream); diff --git a/paddle/operators/net_op.h b/paddle/operators/net_op.h index fcd8134b2c19cae6a4d006a4cd6fe32d2d627c34..2388b094d228562a4c9bfd1ad6840ef1c2068533 100644 --- a/paddle/operators/net_op.h +++ b/paddle/operators/net_op.h @@ -53,16 +53,6 @@ class NetOp : public framework::OperatorBase { this->CompleteAddOp(); } - /** - * Infer all the operators' input and output variables' shapes, will be called - * before every mini-batch - */ - void InferShape(const framework::Scope& scope) const override { - for (auto& op : ops_) { - op->InferShape(scope); - } - } - /** * @brief Run the network. * diff --git a/paddle/operators/net_op_test.cc b/paddle/operators/net_op_test.cc index f2e98ee7a1e14ee739abba01e97608845ce557f4..63bebd5b44719868a38ddf2b023955d1ab05245c 100644 --- a/paddle/operators/net_op_test.cc +++ b/paddle/operators/net_op_test.cc @@ -7,14 +7,12 @@ namespace operators { using Scope = framework::Scope; using DeviceContext = platform::DeviceContext; -static int infer_shape_cnt = 0; static int run_cnt = 0; class TestOp : public framework::OperatorBase { public: using framework::OperatorBase::OperatorBase; DEFINE_OP_CLONE_METHOD(TestOp); - void InferShape(const Scope& scope) const override { ++infer_shape_cnt; } void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const override { ++run_cnt; diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 375d8a35acc0716259071c31bc332fdf5aabce1c..04ebb14f6ee6c73f48aa2f75811a22f9b8a25006 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -24,14 +24,13 @@ class PadOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of PadOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of PadOp should not be null."); - - auto x_dim = ctx.Input("X")->dims(); - auto paddings = Attr>("paddings"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of PadOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of PadOp should not be null."); + + auto x_dim = ctx->GetInputDim("X"); + auto paddings = ctx->Attrs().Get>("paddings"); PADDLE_ENFORCE_EQ(x_dim.size() * 2, int64_t(paddings.size()), "Size of paddings should be equal to 2 * dimension size " "of input tensor."); @@ -39,19 +38,18 @@ class PadOp : public framework::OperatorWithKernel { for (int i = 0; i < x_dim.size(); ++i) { out_dims[i] = x_dim[i] + paddings[i * 2] + paddings[i * 2 + 1]; } - ctx.Output("Out")->Resize( - framework::make_ddim(out_dims)); + ctx->SetOutputDim("Out", framework::make_ddim(out_dims)); if (out_dims[0] == x_dim[0]) { // Only pass LoD when the first dimension is equal between // output and input. - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->ShareLoD("X", /*->*/ "Out"); } } }; class PadOpMaker : public framework::OpProtoAndCheckerMaker { public: - PadOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + PadOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of pad op. " @@ -68,15 +66,15 @@ Given: X = [[1, 2], [3, 4]] -and +and paddings = [0, 1, 1, 2] and - -pad_value = 0 -then we get +pad_value = 0 + +then we get Out = [[0, 1, 2, 0, 0] [0, 3, 4, 0, 0] @@ -101,14 +99,14 @@ class PadOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto *x_g = ctx.Output(framework::GradVarName("X")); - if (x_g != nullptr) { - x_g->Resize(x_dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + auto x_grad_name = framework::GradVarName("X"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); } } }; diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc index 912196c190b5ddbd4e3482a5314e949186b94368..1692464f2833a59243ccc1598422180262a59282 100644 --- a/paddle/operators/prelu_op.cc +++ b/paddle/operators/prelu_op.cc @@ -26,19 +26,14 @@ class PReluOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - auto *in = ctx.Input("X"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Alpha"), - "Input(Alpha) should not be null"); - auto *alpha = ctx.Input("Alpha"); - PADDLE_ENFORCE(alpha->numel() == 1, "Size of weight Alpha must be one."); - - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) should not be null"); - auto *out = ctx.Output("Out"); - out->Resize(in->dims()); - ctx.ShareLoD("X", /*->*/ "Out"); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput("Alpha"), "Input(Alpha) should not be null"); + PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == 1, + "Size of weight Alpha must be one."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null"); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Out"); } }; @@ -68,19 +63,13 @@ class PReluGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto *dx = ctx.Output(framework::GradVarName("X")); - auto *x = ctx.Input("X"); - - auto *dalpha = - ctx.Output(framework::GradVarName("Alpha")); - auto *alpha = ctx.Input("Alpha"); - - dx->Resize(x->dims()); - dalpha->Resize(alpha->dims()); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->SetOutputDim(framework::GradVarName("Alpha"), + ctx->GetInputDim("Alpha")); } }; diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc index 39af08c8751c3b95cf5fdef7395186a0176a20a2..1ba22006f27abc963e7f161636a964863513a40c 100644 --- a/paddle/operators/rank_loss_op.cc +++ b/paddle/operators/rank_loss_op.cc @@ -25,22 +25,21 @@ class RankLossOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { + void InferShape(framework::InferShapeContextBase *ctx) const override { // input check - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), - "Input(Label) shouldn't be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), - "Input(Left) shouldn't be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), - "Input(Right) shouldn't be null"); - auto label_dims = ctx.Input("Label")->dims(); - auto left_dims = ctx.Input("Left")->dims(); - auto right_dims = ctx.Input("Right")->dims(); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null"); + PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null"); + PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null"); + + auto label_dims = ctx->GetInputDim("Label"); + auto left_dims = ctx->GetInputDim("Left"); + auto right_dims = ctx->GetInputDim("Right"); + PADDLE_ENFORCE((label_dims == left_dims) && (left_dims == right_dims), "All inputs must have the same size"); PADDLE_ENFORCE((label_dims.size() == 2) && (label_dims[1] == 1), "All inputs must be row vector with size batch_size x 1."); - ctx.Output("Out")->Resize(label_dims); + ctx->SetOutputDim("Out", label_dims); } }; @@ -91,25 +90,22 @@ class RankLossGradOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), - "Input(Label) shouldn't be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), - "Input(Left) shouldn't be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), - "Input(Right) shouldn't be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) shouldn't be null."); - auto dims = ctx.Input("Left")->dims(); - auto *left_grad = - ctx.Output(framework::GradVarName("Left")); - auto *right_grad = - ctx.Output(framework::GradVarName("Right")); - if (left_grad) { - left_grad->Resize(dims); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto dims = ctx->GetInputDim("Left"); + auto left_grad_name = framework::GradVarName("Left"); + auto right_grad_name = framework::GradVarName("Right"); + + if (ctx->HasOutput(left_grad_name)) { + ctx->SetOutputDim(left_grad_name, dims); } - if (right_grad) { - right_grad->Resize(dims); + + if (ctx->HasOutput(right_grad_name)) { + ctx->SetOutputDim(right_grad_name, dims); } } }; diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index e7deaf9940699b938e4f36358c2c7f3ba15e918b..80de229c333f645fb3098b97fa076c6b77bb7ca9 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -28,29 +28,6 @@ using Variable = framework::Variable; using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -void RecurrentAlgorithm::InferShape(const Scope& scope) const { - auto* input0 = scope.FindVar(arg_->inlinks[0]); - PADDLE_ENFORCE_NOT_NULL(input0); - seq_len_ = input0->GetMutable()->dims()[0]; - PADDLE_ENFORCE_GT(seq_len_, 0); - - CreateScopes(scope); - auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - true /*infer_shape_mode*/); - InitMemories(step_scopes[0], true /*infer_shape_mode*/); - - for (size_t i = 0; i < seq_len_; i++) { - if (i > 0) { - rnn::LinkMemories(step_scopes, arg_->memories, i, -1, - true /*infer_shape_mode*/); - } - (*stepnet_)->InferShape(*step_scopes[i]); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - true /*infer_shape_mode*/); -} - void RecurrentAlgorithm::Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const { auto step_scopes = GetStepScopes(scope); @@ -202,24 +179,6 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients( } } -void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = - scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; - auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - true /*infer_shape_mode*/); - for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) { - if (static_cast(step_id) != seq_len_ - 1) { - rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1, - true /*infer_shape_mode*/); - } - (*stepnet_)->InferShape(*step_scopes[step_id]); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - true /*infer_shape_mode*/); - LinkBootMemoryGradients(step_scopes[0], true /*infer_shape_mode*/); -} - RecurrentGradientOp::RecurrentGradientOp( const std::string& type, const framework::VariableNameMap& inputs, const framework::VariableNameMap& outputs, diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h index ad4df9e55b91dbe89c34762945cd9edefde86e08..c6b9a5533eece9057449b5c875ddcb3cefe716f0 100644 --- a/paddle/operators/recurrent_op.h +++ b/paddle/operators/recurrent_op.h @@ -41,11 +41,6 @@ class RecurrentAlgorithm { stepnet_ = stepnet; } - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const; - protected: /* * The step scopes will be stored in the father scope as a variable. @@ -94,11 +89,6 @@ class RecurrentGradientAlgorithm { void LinkBootMemoryGradients(framework::Scope* step_scopes, bool infer_shape_mode) const; - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const; - protected: inline const std::vector& GetStepScopes( const framework::Scope& scope) const { @@ -124,12 +114,6 @@ class RecurrentOp : public framework::OperatorBase { // TODO(yuyang18): Implement copy ctor well. PADDLE_THROW("Not implemented"); } - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const override { - alg_.InferShape(scope); - } void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const override { @@ -163,13 +147,6 @@ class RecurrentGradientOp : public framework::OperatorBase { PADDLE_THROW("Not Implemented"); } - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const override { - alg_.InferShape(scope); - } - void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const override { alg_.Run(scope, dev_ctx); diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index ddb93007e21e4d1ae4be3650019c8bc6a680252d..a3c3fa2716ad9f6487e3eff2d98b2c76d964ddef 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -26,14 +26,14 @@ class ReshapeOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { + void InferShape(framework::InferShapeContextBase *ctx) const override { // input check - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of ReshapeOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of ReshapeOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of ReshapeOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ReshapeOp should not be null."); - auto shape = ctx.Attr>("shape"); + auto shape = ctx->Attrs().Get>("shape"); PADDLE_ENFORCE(shape.size() > 0, "Attr(shape) shouldn't be empty."); for (auto dim : shape) { PADDLE_ENFORCE(dim > 0, "Each dimension of shape must be positive."); @@ -41,8 +41,8 @@ class ReshapeOp : public framework::OperatorWithKernel { // capacity check int64_t capacity = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - auto *in = ctx.Input("X"); - int64_t in_size = framework::product(in->dims()); + auto x_dims = ctx->GetInputDim("X"); + int64_t in_size = framework::product(x_dims); PADDLE_ENFORCE_EQ(capacity, in_size, "The size of Input(X) mismatches with Attr(shape)."); // resize output @@ -50,11 +50,11 @@ class ReshapeOp : public framework::OperatorWithKernel { std::transform(shape.begin(), shape.end(), shape_int64.begin(), [](int a) { return static_cast(a); }); auto out_dims = framework::make_ddim(shape_int64); - ctx.Output("Out")->Resize(out_dims); - if (shape[0] == in->dims()[0]) { + ctx->SetOutputDim("Out", out_dims); + if (shape[0] == x_dims[0]) { // Only pass LoD when the first dimension is equal between // output and input. - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->ShareLoD("X", /*->*/ "Out"); } } }; @@ -76,7 +76,7 @@ Given a 2-D tensor X with 2 rows and 2 columns [[1, 2], [3, 4]] -with target shape = [1, 4], the reshape operator will transform +with target shape = [1, 4], the reshape operator will transform the tensor X into a 1-D tensor: [1, 2, 3, 4] @@ -94,13 +94,11 @@ class ReshapeGradOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) shouldn't be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) shouldn't be null."); - auto dims = ctx.Input("X")->dims(); - auto *d_in = ctx.Output(framework::GradVarName("X")); - d_in->Resize(dims); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } }; diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index fc3ad721f210213491617452141dfa8834b067c0..1fcf0959dffd6a68d97dec4e2b5b509d06c0d09c 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -24,16 +24,16 @@ class RowwiseAddOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of RowwiseAddOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("b"), - "Input(b) of RowwiseAddOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of RowwiseAddOp should not be null."); - - auto x_dims = ctx.Input("X")->dims(); - auto b_dims = ctx.Input("b")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of RowwiseAddOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("b"), + "Input(b) of RowwiseAddOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of RowwiseAddOp should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + auto b_dims = ctx->GetInputDim("b"); PADDLE_ENFORCE_GT( x_dims.size(), b_dims.size(), "The rank of input `X` must be larger than the one of input `b`."); @@ -43,16 +43,17 @@ class RowwiseAddOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims, "The width of two operands must be same"); - PADDLE_ENFORCE_EQ(ctx.OutputSize("Out"), 1, "The output size must be 1"); - ctx.Output("Out")->Resize(x_dims); - ctx.ShareLoD("X", /*->*/ "Out"); + PADDLE_ENFORCE_EQ(ctx->Outputs("Out").size(), 1, + "The output size must be 1"); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", /*->*/ "Out"); } }; class RowwiseAddOpMaker : public framework::OpProtoAndCheckerMaker { public: - RowwiseAddOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + RowwiseAddOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The left input of row-wise add op, must be matrix"); AddInput("b", "The right input of row-wise add op, must be vector"); @@ -69,25 +70,29 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("b"), "b should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto b_dims = ctx.Input("b")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "X should not be null"); + PADDLE_ENFORCE(ctx->HasInput("b"), "b should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + auto b_dims = ctx->GetInputDim("b"); PADDLE_ENFORCE_GT( x_dims.size(), b_dims.size(), "The rank of input `X` must be larger than the one of input `b`."); - int num_col_dims = x_dims.size() - b_dims.size(); + int64_t num_col_dims = x_dims.size() - b_dims.size(); PADDLE_ENFORCE_EQ( framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims, "The width of two operands must be same"); - auto *dx = ctx.Output(framework::GradVarName("X")); - auto *db = ctx.Output(framework::GradVarName("b")); - if (dx) dx->Resize(x_dims); - if (db) db->Resize(b_dims); + auto x_grad_name = framework::GradVarName("X"); + auto b_grad_name = framework::GradVarName("b"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(b_grad_name)) { + ctx->SetOutputDim(b_grad_name, b_dims); + } } }; diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 1ae77a9722ef1a5548a6c4100c32fdddcee8c5cd..e92501e12834b92875f494de401672344f50e3b5 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -26,16 +26,13 @@ class ScaleOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of ScaleOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of ScaleOp should not be null."); - - auto *in = ctx.Input("X"); - auto *out = ctx.Output("Out"); - out->Resize(in->dims()); - ctx.ShareLoD("X", /*->*/ "Out"); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of ScaleOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ScaleOp should not be null."); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Out"); } }; diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc index 3f02081a060281dec533c02b346f0667da28b8c3..3fc4a39ebc5526bfed61ba667c3cdc214cdd056c 100644 --- a/paddle/operators/scatter_op.cc +++ b/paddle/operators/scatter_op.cc @@ -23,29 +23,30 @@ class ScatterOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Ref"), - "Input(Ref) of ScatterOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Index"), - "Input(Index) of ScatterOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Updates"), - "Input(Updates) of ScatterOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of ScatterOp should not be null."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Ref"), + "Input(Ref) of ScatterOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Index"), + "Input(Index) of ScatterOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Updates"), + "Input(Updates) of ScatterOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ScatterOp should not be null."); - PADDLE_ENFORCE_EQ(ctx.Input("Index")->dims().size(), 1, + auto updates_dims = ctx->GetInputDim("Updates"); + auto ref_dims = ctx->GetInputDim("Ref"); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("Index").size(), 1, "Update Index should be 1-D."); - PADDLE_ENFORCE_EQ(ctx.Input("Ref")->dims().size(), - ctx.Input("Updates")->dims().size(), + PADDLE_ENFORCE_EQ(ref_dims.size(), updates_dims.size(), "Reference and Updates should have the same shape size"); - PADDLE_ENFORCE_EQ(ctx.Input("Updates")->dims()[0], - ctx.Input("Index")->dims()[0], + PADDLE_ENFORCE_EQ(ctx->GetInputDim("Updates")[0], + ctx->GetInputDim("Index")[0], "Updates and Index should have same batch-size."); - framework::DDim data_dim(ctx.Input("Updates")->dims()); - for (int i = 1; i < data_dim.size(); ++i) - PADDLE_ENFORCE_EQ(data_dim[i], ctx.Input("Updates")->dims()[i]); - ctx.Output("Out")->Resize( - ctx.Input("Ref")->dims()); + framework::DDim data_dim(updates_dims); + for (int i = 1; i < data_dim.size(); ++i) { + PADDLE_ENFORCE_EQ(data_dim[i], updates_dims[i]); + } + ctx->SetOutputDim("Out", ref_dims); } }; @@ -54,22 +55,17 @@ class ScatterGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - auto *dUpdates = - ctx.Output(framework::GradVarName("Updates")); - auto *Updates = ctx.Input("Updates"); - auto *dRef = ctx.Output(framework::GradVarName("Ref")); - auto *Ref = ctx.Input("Ref"); - - dRef->Resize(Ref->dims()); - dUpdates->Resize(Updates->dims()); + void InferShape(framework::InferShapeContextBase* ctx) const override { + ctx->SetOutputDim(framework::GradVarName("Updates"), + ctx->GetInputDim("Updates")); + ctx->SetOutputDim(framework::GradVarName("Ref"), ctx->GetInputDim("Ref")); } }; class ScatterOpMaker : public framework::OpProtoAndCheckerMaker { public: - ScatterOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + ScatterOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Ref", "The source input of scatter op"); AddInput("Index", @@ -77,13 +73,14 @@ class ScatterOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Updates", "The updated value of updates op"); AddOutput("Out", "The output of add op"); AddComment(R"DOC( -Scatter Operator by selecting from the first axis, +Scatter Operator by selecting from the first axis, Out = Ref Out[Index] = Ref[Index] + Updates )DOC"); } }; + } // namespace operators } // namespace paddle diff --git a/paddle/operators/sequence_pool_op.cc b/paddle/operators/sequence_pool_op.cc index 73f9cb879a2ef690909428b3b672b12717a6a02c..17685ea654715f6996e17f6228f266c3aa1ee424 100644 --- a/paddle/operators/sequence_pool_op.cc +++ b/paddle/operators/sequence_pool_op.cc @@ -22,23 +22,12 @@ class SequencePoolOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of SequencePoolOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of SequencePoolOp should not be null."); - - auto* x = ctx.Input("X"); - auto dims = x->dims(); - auto lod = x->lod(); - PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); - PADDLE_ENFORCE_GE( - dims[0], - /*batch size = */ static_cast(lod[0].size() - 1), - "The first dimension of Input(X) must be large than batch size."); - dims[0] = lod[0].size() - 1; - ctx.Output("Out")->Resize({dims}); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SequenceAvgPoolOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SequenceAvgPoolOp should not be null."); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); } }; @@ -61,17 +50,17 @@ class SequencePoolOpMaker : public framework::OpProtoAndCheckerMaker { SequencePoolOp pools features of all time-steps of each instance. For a mini-batch of 3 variable lengths sentences, containing 2, 3, and 2 time-steps: - + Assume X is a [7,M,N] float LoDTensor, and X->lod()[0] = [0, 2, 5, 7]. - Besides, for the sake of simplicity, we assume M=1 and N=1, + Besides, for the sake of simplicity, we assume M=1 and N=1, and the value of X = [[1, 3], [2, 4, 6], [5, 1]]. Thus, Out is a [3,1,1] float LoDTensor, but Out->lod() is nullptr. - And for different strategy, the value of Out is as follows: + And for different strategy, the value of Out is as follows: - AVERAGE: [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 - SUM: [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 - - SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), + - SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), 6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2) - MAX: [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) - LAST: [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) @@ -85,22 +74,18 @@ class SequencePoolGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Gradient of Out should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "The input X should not be null."); - auto og_dims = - ctx.Input(framework::GradVarName("Out"))->dims(); - auto x_dims = ctx.Input("X")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Gradient of Out should not be null."); + PADDLE_ENFORCE(ctx->HasInput("X"), "The input X should not be null."); + auto og_dims = ctx->GetInputDim(framework::GradVarName("Out")); + auto x_dims = ctx->GetInputDim("X"); PADDLE_ENFORCE_EQ(og_dims.size(), x_dims.size(), "The rank of output grad must equal to Input(X)."); for (int64_t i = 1; i < og_dims.size(); ++i) { PADDLE_ENFORCE_EQ(og_dims[i], x_dims[i], "The dimension mismatch."); } - auto* x_grad = - ctx.Output(framework::GradVarName("X")); - x_grad->Resize(x_dims); + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); } }; diff --git a/paddle/operators/sequence_pool_op.h b/paddle/operators/sequence_pool_op.h index 231614b4c1cb0eb1901b1720e933aed5cbb25f77..cb80586e88f8d9e31b7b91a54f5e05ac6fa73f0f 100644 --- a/paddle/operators/sequence_pool_op.h +++ b/paddle/operators/sequence_pool_op.h @@ -46,16 +46,27 @@ class SequencePoolKernel : public framework::OpKernel { int strategy = context.Attr("strategy"); auto dims = in->dims(); - auto lod = in->lod()[0]; + auto lod = in->lod(); int64_t w = in->numel() / dims[0]; + // InferShape by lod + PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); + PADDLE_ENFORCE_GE( + dims[0], + /*batch size = */ static_cast(lod[0].size() - 1), + "The first dimension of Input(X) must be large than batch size."); + dims[0] = lod[0].size() - 1; + out->Resize({dims}); + + auto lod_level_0 = lod[0]; + out->mutable_data(context.GetPlace()); auto place = context.GetEigenDevice(); - for (int i = 0; i < static_cast(lod.size()) - 1; ++i) { - Tensor in_t = - in->Slice(static_cast(lod[i]), static_cast(lod[i + 1])); + for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { + Tensor in_t = in->Slice(static_cast(lod_level_0[i]), + static_cast(lod_level_0[i + 1])); Tensor out_t = out->Slice(i, i + 1); - int64_t h = static_cast(lod[i + 1] - lod[i]); + int64_t h = static_cast(lod_level_0[i + 1] - lod_level_0[i]); auto in_e = EigenMatrix::From(in_t, framework::make_ddim({h, w})); auto out_e = EigenVector::Flatten(out_t); diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index b063e2427217f20eb89f7cd1af0354ad0e400feb..3bce95535cf10c0df95b503c6e362b3f0ba2e723 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -22,19 +22,18 @@ class SGDOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("param"), - "Input(param) of SGDOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("grad"), - "Input(grad) of SGDOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("param_out"), - "Output(param_out) of SGDOp should not be null."); - - PADDLE_ENFORCE_EQ(ctx.Input("param")->dims(), - ctx.Input("grad")->dims(), + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("param"), + "Input(param) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("grad"), + "Input(grad) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("param_out"), + "Output(param_out) of SGDOp should not be null."); + + auto param_dim = ctx->GetInputDim("param"); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("grad"), "Two input of SGD Op's dimension must be same."); - ctx.Output("param_out") - ->Resize(ctx.Input("param")->dims()); + ctx->SetOutputDim("param_out", param_dim); } }; diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index ae6d1c80b300690b070024d6266a1b99bf2ef04f..2d197e3b1b763fa87939623d47728aab3bff7cd1 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -22,33 +22,28 @@ class SmoothL1LossOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X must be initialized."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Y must be initialized."); - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - PADDLE_ENFORCE_EQ(x->dims(), y->dims(), - "The shape of X and Y must be the same."); - PADDLE_ENFORCE_GE(x->dims().size(), 2, + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "X must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Y must be initialized."); + + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + PADDLE_ENFORCE_EQ(x_dims, y_dims, "The shape of X and Y must be the same."); + PADDLE_ENFORCE_GE(x_dims.size(), 2, "The tensor rank of X must be at least 2."); - auto* inside_weight = ctx.Input("InsideWeight"); - if (inside_weight) { - auto* outside_weight = ctx.Input("OutsideWeight"); - PADDLE_ENFORCE_NOT_NULL(outside_weight, - "If weights are provided, must specify both " - "inside and outside weights."); - PADDLE_ENFORCE_EQ(inside_weight->dims(), x->dims(), + if (ctx->HasInput("InsideWeight")) { + PADDLE_ENFORCE(ctx->HasInput("OutsideWeight"), + "If weights are provided, must specify both " + "inside and outside weights."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("InsideWeight"), x_dims, "The shape of InsideWeight must be same as X."); - PADDLE_ENFORCE_EQ(outside_weight->dims(), x->dims(), + PADDLE_ENFORCE_EQ(ctx->GetInputDim("OutsideWeight"), x_dims, "The shape of OutsideWeight must be same as X."); } - auto* diff = ctx.Output("Diff"); - auto* out = ctx.Output("Out"); - diff->Resize(x->dims()); + ctx->SetOutputDim("Diff", x_dims); // loss is a two-rank tensor - out->Resize({x->dims()[0], 1}); + ctx->SetOutputDim("Out", {x_dims[0], 1}); } }; @@ -99,12 +94,9 @@ class SmoothL1LossGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - auto in_dims = ctx.Input("X")->dims(); - auto out_dims = - ctx.Input(framework::GradVarName("Out"))->dims(); - auto* x_grad = ctx.Output(framework::GradVarName("X")); - auto* y_grad = ctx.Output(framework::GradVarName("Y")); + void InferShape(framework::InferShapeContextBase* ctx) const override { + auto in_dims = ctx->GetInputDim("X"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); PADDLE_ENFORCE_GE(out_dims.size(), 2, "The tensor rank of Input(Out@Grad) should be 2."); @@ -114,8 +106,14 @@ class SmoothL1LossGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(out_dims[1], 1, "The 2nd dimension of Input(Out@Grad) must be 1."); - if (x_grad) x_grad->Resize(in_dims); - if (y_grad) y_grad->Resize(in_dims); + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, in_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, in_dims); + } } }; diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index e15cfe485016552971924a40a172e74a90629dce..e353afee3e10247fbd5c7f4282c366cd1bc39552 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -22,22 +22,23 @@ class SoftmaxOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of SoftmaxOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) of SoftmaxOp should not be null."); - - PADDLE_ENFORCE(ctx.Input("X")->dims().size() == 2UL, + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SoftmaxOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Y"), + "Output(Y) of SoftmaxOp should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + PADDLE_ENFORCE(x_dims.size() == 2UL, "The input of softmax op must be a matrix."); - ctx.Output("Y")->Resize(ctx.Input("X")->dims()); + ctx->SetOutputDim("Y", x_dims); } }; class SoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { public: - SoftmaxOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + SoftmaxOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input tensor of softmax. " @@ -68,16 +69,15 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) should be not null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Y")), - "Input(Y@GRAD) should be not null."); - PADDLE_ENFORCE_EQ(ctx.Input("Y")->dims(), - ctx.Input(framework::GradVarName("Y"))->dims(), + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should be not null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Y")), + "Input(Y@GRAD) should be not null."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("Y"), + ctx->GetInputDim(framework::GradVarName("Y")), "Input(Y) and its gradients should have a same shape."); - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("X")->dims()); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } }; diff --git a/paddle/operators/softmax_op.h b/paddle/operators/softmax_op.h index 8a3a5ab927c0e2937936fcc973f000d4d95c3dbc..7220f486be055e1b841a06b15f519717c54f575c 100644 --- a/paddle/operators/softmax_op.h +++ b/paddle/operators/softmax_op.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/softmax.h" namespace paddle { namespace operators { @@ -30,36 +31,11 @@ class SoftmaxKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& context) const override { auto X = context.Input("X"); auto Y = context.Output("Y"); - Y->mutable_data(context.GetPlace()); - - auto logits = EigenMatrix::From(*X); - auto softmax = EigenMatrix::From(*Y); - - const int kBatchDim = 0; - const int kClassDim = 1; - const int batch_size = logits.dimension(kBatchDim); - const int num_classes = logits.dimension(kClassDim); - - Eigen::DSizes along_class(kClassDim); - Eigen::DSizes batch_by_one(batch_size, 1); - Eigen::DSizes one_by_class(1, num_classes); - - auto shifted_logits = (logits - - logits.maximum(along_class) - .eval() - .reshape(batch_by_one) - .broadcast(one_by_class)); - - softmax.device(context.GetEigenDevice()) = shifted_logits.exp(); + // allocate memory on device. + Y->mutable_data(context.GetPlace()); - softmax.device(context.GetEigenDevice()) = - (softmax * - softmax.sum(along_class) - .inverse() - .eval() - .reshape(batch_by_one) - .broadcast(one_by_class)); + math::SoftmaxFunctor()(context, X, Y); } }; @@ -67,8 +43,6 @@ template class SoftmaxGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - std::shared_ptr scale_ = std::make_shared(); - auto Y = context.Input("Y"); auto dY = context.Input(framework::GradVarName("Y")); auto dX = context.Output(framework::GradVarName("X")); diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..e2299b254458cdd42dee4683561d4d5c81653fb1 --- /dev/null +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -0,0 +1,166 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/softmax_with_cross_entropy_op.h" + +namespace paddle { +namespace operators { + +class SoftmaxWithCrossEntropyOpMaker + : public framework::OpProtoAndCheckerMaker { + public: + SoftmaxWithCrossEntropyOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Logits", + "(Tensor, default: Tensor), The unscaled log probabilities " + "which is a 2-D tensor with shape [N x K]. N is the batch_size, " + "and K is the class number.") + .NotInGradient(); + AddInput( + "Label", + "(Tensor, default: Tensor), The ground truth which is a 2-D " + "tensor. " + "If softLable is set to 0, Label is a Tensor with shape [N x 1]. " + "If softLable is set to 1, Label is a Tensor " + "with shape [N x K]."); + AddOutput( + "Softmax", + "(Tensor, default: Tensor), A 2-D tensor with shape [N x K]. " + "The outputs value of softmax activation by given the input batch, " + "which will be used in backward calculation.") + .AsIntermediate(); + AddOutput("Loss", + "(Tensor, default: Tensor), A 2-D tensor. The cross " + "entropy loss with shape [N x 1]."); + AddAttr( + "softLabel", + "(bool, default: false), A flag to indicate whether to interpretate " + "the given labels as soft labels.") + .SetDefault(false); + AddComment(R"DOC( +Cross entropy loss with softmax are used as the output layer extensively. This +operator computes the softmax normalized values for each row of the input +tensor, after which cross-entropy loss is then computed. This provides a more +numerically stable gradient. + +Because this operators performs a softmax on logits internally, it expects +unscaled logits. Please do not call this op with the output of softmax operator, +which will produce incorrect results. + +This operators expects mutually exclusive hard labels, each sample in a batch +is in exactly one class with probabilities 1. Each sample in the batch with one +and only one label. + +Equation: + +1) hard label (one-hot label) + +Loss_j = -\text{Logit}_{Label_j} + \log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right), j = 1, ..., K + +2) soft label (a distribution over all classes) + +Loss_j = -\sum_{i=0}^{K}\text{Label}_i\left(\text{Logit}_i-\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right)\right), j = 1,...,K + +)DOC"); + } +}; + +class SoftmaxWithCrossEntropyOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Logits"), + "Input(Logits) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null."); + + PADDLE_ENFORCE(ctx->HasOutput("Softmax"), + "Output(Softmax) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("Loss"), "Output(Loss) should be not null."); + + auto logits_dims = ctx->GetInputDim("Logits"); + auto labels_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE_EQ( + logits_dims.size(), 2UL, + "The input of softmax_with_cross_entropy should be a 2-D tensor."); + PADDLE_ENFORCE_EQ(labels_dims.size(), 2UL, + "The labels should be a 2-D tensor."); + + if (ctx->Attrs().Get("softLabel")) { + PADDLE_ENFORCE_EQ(logits_dims[1], labels_dims[1], + "If Attr(softLabel) == true, the 2nd dimension of " + "Input(X) and Input(Label) should be equal."); + } else { + PADDLE_ENFORCE_EQ(labels_dims[1], 1UL, + "If Attr(softLabel) == false, the 2nd dimension of " + "Input(Label) should be 1."); + } + + ctx->SetOutputDim("Softmax", logits_dims); + ctx->SetOutputDim("Loss", {logits_dims[0], 1}); + + ctx->ShareLoD("Logits", /*->*/ "Softmax"); + ctx->ShareLoD("Logits", /*->*/ "Loss"); + } +}; + +class SoftmaxWithCrossEntropyOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Loss")), + "Input(Loss@Grad) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Softmax"), + "Input(Softmax) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("Logits")), + "Output(Logits@Grad) should be not null."); + + auto softmax_dims = ctx->GetInputDim("Softmax"); + auto labels_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE_EQ(labels_dims.size(), 2UL, + "The labels should be a 2-D tensor."); + + if (ctx->Attrs().Get("softLabel")) { + PADDLE_ENFORCE_EQ(softmax_dims[1], labels_dims[1], + "When Attr(softLabel) == true, the 2nd dimension of " + "Input(X) and Input(Label) should be equal."); + } else { + PADDLE_ENFORCE_EQ(labels_dims[1], 1UL, + "When Attr(softLabel) == false, the 2nd dimension of " + "Input(Label) should be 1."); + } + + ctx->SetOutputDim(framework::GradVarName("Logits"), + ctx->GetInputDim("Softmax")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp, + ops::SoftmaxWithCrossEntropyOpMaker, + softmax_with_cross_entropy_grad, + ops::SoftmaxWithCrossEntropyOpGrad); +REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy, + ops::SoftmaxWithCrossEntropyKernel); +REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy_grad, + ops::SoftmaxWithCrossEntropyGradKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.cu b/paddle/operators/softmax_with_cross_entropy_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..1cf4296dccf68aece6fdfb7910a9c68449633b76 --- /dev/null +++ b/paddle/operators/softmax_with_cross_entropy_op.cu @@ -0,0 +1,119 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU + +#include "paddle/operators/softmax_with_cross_entropy_op.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +namespace { +template +__global__ void CrossEntropyGrad(T* out_grad, const T* in_grad, + const int* labels, const int batch_size, + const int class_num) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int sample_idx = tid / class_num; + + if (tid < batch_size * class_num) out_grad[tid] *= in_grad[sample_idx]; + __syncthreads(); + + if (tid < batch_size) { + PADDLE_ASSERT(labels[sample_idx] >= 0 && labels[sample_idx] < class_num); + out_grad[tid * class_num + labels[tid]] -= 1.; + } +} + +template +__global__ void SoftCrossEntropyGradientKernel(T* logit_grad, + const T* loss_grad, + const T* labels, + const int batch_size, + const int class_num) { + int ids = blockIdx.x * blockDim.x + threadIdx.x; + if (ids < batch_size * class_num) { + int row_ids = ids / class_num; + logit_grad[ids] = logit_grad[ids] * loss_grad[row_ids] - labels[ids]; + } +} +} // namespace + +template +class SoftmaxWithCrossEntropyCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + PADDLE_ENFORCE(platform::is_gpu_place(context.GetPlace()), + "This kernel only runs on GPU device."); + const Tensor* logits = context.Input("Logits"); + const Tensor* labels = context.Input("Label"); + Tensor* softmax = context.Output("Softmax"); + + Tensor* loss = context.Output("Loss"); + softmax->mutable_data(context.GetPlace()); + loss->mutable_data(context.GetPlace()); + + math::SoftmaxFunctor()(context, logits, softmax); + math::CrossEntropyFunctor()( + context, loss, softmax, labels, context.Attr("softLabel")); + } +}; + +template +class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + PADDLE_ENFORCE(platform::is_gpu_place(context.GetPlace()), + "This kernel only runs on GPU device."); + const Tensor* labels = context.Input("Label"); + const T* loss_grad_data = + context.Input(framework::GradVarName("Loss"))->data(); + Tensor* logit_grad = + context.Output(framework::GradVarName("Logits")); + logit_grad->ShareDataWith(*context.Input("Softmax")); + T* logit_grad_data = logit_grad->data(); + + const int batch_size = logit_grad->dims()[0]; + const int class_num = logit_grad->dims()[1]; + int block = 512; + int grid = (batch_size * class_num + block - 1) / block; + + if (context.Attr("softLabel")) { + const T* label_data = labels->data(); + SoftCrossEntropyGradientKernel<<< + grid, block, 0, reinterpret_cast( + context.device_context()) + .stream()>>>(logit_grad_data, loss_grad_data, + label_data, batch_size, class_num); + } else { + const int* label_data = labels->data(); + CrossEntropyGrad<<< + grid, block, 0, reinterpret_cast( + context.device_context()) + .stream()>>>(logit_grad_data, loss_grad_data, + label_data, batch_size, class_num); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy, + ops::SoftmaxWithCrossEntropyCUDAKernel); +REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy_grad, + ops::SoftmaxWithCrossEntropyGradCUDAKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.h b/paddle/operators/softmax_with_cross_entropy_op.h new file mode 100644 index 0000000000000000000000000000000000000000..bf792c1f59e2e43a98c93bddbc2aa63d646dee6f --- /dev/null +++ b/paddle/operators/softmax_with_cross_entropy_op.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/cross_entropy.h" +#include "paddle/operators/math/softmax.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +template +class SoftmaxWithCrossEntropyKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + PADDLE_ENFORCE(platform::is_cpu_place(context.GetPlace()), + "This kernel only runs on CPU."); + const Tensor* logits = context.Input("Logits"); + const Tensor* labels = context.Input("Label"); + Tensor* softmax = context.Output("Softmax"); + Tensor* loss = context.Output("Loss"); + + softmax->mutable_data(context.GetPlace()); + loss->mutable_data(context.GetPlace()); + + math::SoftmaxFunctor()(context, logits, softmax); + math::CrossEntropyFunctor()( + context, loss, softmax, labels, context.Attr("softLabel")); + } +}; + +template +class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* out_grad = + context.Input(framework::GradVarName("Loss")); + const Tensor* labels = context.Input("Label"); + Tensor* logit_grad = + context.Output(framework::GradVarName("Logits")); + logit_grad->ShareDataWith(*context.Input("Softmax")); + + const int class_num = logit_grad->dims()[1]; + if (context.Attr("softLabel")) { + auto out_grad_mat = EigenMatrix::From(*out_grad); + auto logit_grad_mat = EigenMatrix::From(*logit_grad); + auto lbl_mat = EigenMatrix::From(*labels); + + logit_grad_mat.device(context.GetEigenDevice()) = + logit_grad_mat * + out_grad_mat.broadcast(Eigen::DSizes(1, class_num)) - + lbl_mat; + } else { + const int batch_size = logit_grad->dims()[0]; + const int* label_data = labels->data(); + const T* out_grad_data = out_grad->data(); + T* logit_grad_data = logit_grad->data(); + + for (int i = 0; i < batch_size; ++i) { + int index = i * class_num + label_data[i]; + logit_grad_data[index] = + (out_grad_data[i] * logit_grad_data[index] - 1.); + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc index a9d35b4fb79ae83379552ae2c2b4d694bd8f86dd..5f4b5539affef6fe1d3c4d15fff77d983b5e107f 100644 --- a/paddle/operators/split_op.cc +++ b/paddle/operators/split_op.cc @@ -24,40 +24,43 @@ class SplitOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - // infershape - auto *in = ctx.Input("X"); - auto outs = ctx.MultiOutput("Out"); - size_t axis = static_cast(ctx.Attr("axis")); - size_t num = static_cast(ctx.Attr("num")); - std::vector sections = - static_cast>(ctx.Attr>("sections")); - const size_t n = outs.size(); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SplitOp should not be null."); + PADDLE_ENFORCE_GE(ctx->Outputs("Out").size(), 1UL, + "Outputs(Out) of SplitOp should not be empty."); + auto in_dims = ctx->GetInputDim("X"); + auto outs_names = ctx->Outputs("Out"); + size_t axis = static_cast(ctx->Attrs().Get("axis")); + size_t num = static_cast(ctx->Attrs().Get("num")); + std::vector sections = static_cast>( + ctx->Attrs().Get>("sections")); + const size_t outs_number = outs_names.size(); + std::vector outs_dims; + outs_dims.reserve(outs_number); if (num > 0) { - int64_t in_axis_dim = in->dims()[axis]; + int64_t in_axis_dim = in_dims[axis]; PADDLE_ENFORCE_EQ(in_axis_dim % num, 0, "tensor split does not result" " in an equal division"); size_t out_axis_dim = in_axis_dim / num; - for (size_t i = 0; i < n; ++i) { - auto dim = in->dims(); + for (size_t i = 0; i < outs_number; ++i) { + auto dim = in_dims; dim[axis] = out_axis_dim; - outs[i]->Resize(dim); + outs_dims.push_back(dim); } } else if (sections.size() > 0) { - PADDLE_ENFORCE_EQ(sections.size(), n, + PADDLE_ENFORCE_EQ(sections.size(), outs_number, "tensor split sections size" "should be equal to output size."); - for (size_t i = 0; i < n; ++i) { - auto dim = in->dims(); + for (size_t i = 0; i < outs_number; ++i) { + auto dim = in_dims; dim[axis] = sections[i]; - outs[i]->Resize(dim); + outs_dims.push_back(dim); } - } else { - PADDLE_ENFORCE_NOT_NULL(nullptr, "split operator should", - " specify indices or sections."); } + ctx->SetOutputsDim("Out", outs_dims); } }; @@ -115,4 +118,4 @@ USE_CPU_ONLY_OP(concat); REGISTER_OP(split, ops::SplitOp, ops::SplitOpMaker, split_grad, ops::SplitOpGrad); REGISTER_OP_CPU_KERNEL(split, - ops::SplitKernel); + ops::SplitOpKernel); diff --git a/paddle/operators/split_op.cu b/paddle/operators/split_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..93d1fc3c44cbc146c945c51af1abe6494572d1ae --- /dev/null +++ b/paddle/operators/split_op.cu @@ -0,0 +1,18 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/split_op.h" +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(split, + ops::SplitOpKernel); diff --git a/paddle/operators/split_op.h b/paddle/operators/split_op.h index 860690ee895075fda9ddef08776a2102642efff9..8ab8e0ee4fea621b34da73507c53846100d61a17 100644 --- a/paddle/operators/split_op.h +++ b/paddle/operators/split_op.h @@ -16,44 +16,29 @@ limitations under the License. */ #include #include "paddle/framework/op_registry.h" +#include "paddle/operators/strided_memcpy.h" namespace paddle { namespace operators { template -class SplitKernel : public framework::OpKernel { +class SplitOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { auto* in = ctx.Input("X"); auto outs = ctx.MultiOutput("Out"); + auto in_stride = framework::stride(in->dims()); int64_t axis = static_cast(ctx.Attr("axis")); - size_t before = 1, after = 1; const size_t n = outs.size(); - size_t input_axis_dim = in->dims()[axis]; - - for (int64_t i = 0; i < in->dims().size(); ++i) { - if (i == axis) { - continue; - } - if (i < axis) { - before *= in->dims()[i]; - } else { - after *= in->dims()[i]; - } - } size_t input_offset = 0; for (size_t i = 0; i < n; i++) { auto& out = outs[i]; + out->mutable_data(ctx.GetPlace()); size_t axis_dim = out->dims()[axis]; - for (size_t j = 0; j < before; j++) { - size_t len = axis_dim * after * sizeof(T); - T* dest = - out->mutable_data(platform::CPUPlace()) + axis_dim * after * j; - const T* src = - in->data() + input_offset + input_axis_dim * after * j; - memcpy(dest, src, len); - } - input_offset += axis_dim * after; + auto out_stride = framework::stride(out->dims()); + StridedMemcpy(ctx.device_context(), in->data() + input_offset, + in_stride, out->dims(), out_stride, out->data()); + input_offset += axis_dim * in_stride[axis]; } } }; diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc index 33a564b05b1b490c6d23b7d17cef45b7740dfa39..5a0cb596008a98aacf5e7b5ff70307ea1b8508e6 100644 --- a/paddle/operators/squared_l2_distance_op.cc +++ b/paddle/operators/squared_l2_distance_op.cc @@ -22,24 +22,19 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("X"), - "Input(X) of SquaredL2DistanceOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("Y"), - "Input(Y) of SquaredL2DistanceOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("sub_result"), + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SquaredL2DistanceOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), + "Input(Y) of SquaredL2DistanceOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("sub_result"), "Output(sub_result) of SquaredL2DistanceOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of SquaredL2DistanceOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SquaredL2DistanceOp should not be null."); - auto* x = ctx.Input("X"); - auto x_dims = x->dims(); - auto* y = ctx.Input("Y"); - auto y_dims = y->dims(); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); PADDLE_ENFORCE_EQ(framework::arity(x_dims), framework::arity(y_dims), "Tensor rank of both SquaredL2DistanceOp's " @@ -47,17 +42,16 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel { int rank = framework::arity(x_dims); PADDLE_ENFORCE_GE(rank, 2, "Tensor rank should be at least equal to 2."); - PADDLE_ENFORCE_EQ(x->numel() / x_dims[0], y->numel() / y_dims[0], + PADDLE_ENFORCE_EQ(product(x_dims) / x_dims[0], product(y_dims) / y_dims[0], "Product of dimensions expcet the first dimension of " "input and target must be equal."); PADDLE_ENFORCE(y_dims[0] == 1 || y_dims[0] == x_dims[0], "First dimension of target must be equal to input " "or to 1."); - ctx.Output("sub_result") - ->Resize({x_dims[0], x->numel() / x_dims[0]}); - ctx.Output("Out")->Resize({x_dims[0], 1}); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("sub_result", {x_dims[0], product(x_dims) / x_dims[0]}); + ctx->SetOutputDim("Out", {x_dims[0], 1}); + ctx->ShareLoD("X", /*->*/ "Out"); } }; @@ -92,22 +86,22 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Gradient of Out should not be null"); - auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Gradient of Out should not be null"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); PADDLE_ENFORCE_EQ(out_dims[0], x_dims[0], "First dimension of output gradient and " "input value must be equal."); PADDLE_ENFORCE_EQ(out_dims[1], 1, "Second dimension of output gradient " "must be 1."); - auto* x_grad = ctx.Output(framework::GradVarName("X")); - auto* y_grad = ctx.Output(framework::GradVarName("Y")); - if (x_grad) x_grad->Resize(x_dims); - if (y_grad) y_grad->Resize(y_dims); + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + if (ctx->HasOutput(x_grad_name)) ctx->SetOutputDim(x_grad_name, x_dims); + if (ctx->HasOutput(y_grad_name)) ctx->SetOutputDim(y_grad_name, y_dims); } }; diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 437fc262f359525045a4d772ee2c204ef571caa7..8f62a9f4db8d39edc11949df513aebf4fa257d45 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -21,31 +21,27 @@ class SumOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE(!ctx.MultiInputVar("X").empty(), - "Input(X) of SumOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of SumOp should not be null."); - - auto ins = ctx.MultiInput("X"); - auto *out = ctx.Output("Out"); - int N = ins.size(); - - auto in_dim = ins[0]->dims(); + void InferShape(framework::InferShapeContextBase* ctx) const override { + auto x_dims = ctx->GetInputsDim("X"); + PADDLE_ENFORCE(!x_dims.empty(), "Input(X) of SumOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SumOp should not be null."); + auto in_dim = x_dims[0]; + size_t N = x_dims.size(); PADDLE_ENFORCE_GT(N, 1, "Input tensors count should > 1."); - for (int i = 1; i < N; i++) { - auto dim = ins[i]->dims(); + for (size_t i = 1; i < N; i++) { + auto dim = x_dims[i]; PADDLE_ENFORCE(in_dim == dim, "Input tensors must have same shape"); } - out->Resize(in_dim); - ctx.ShareLoD("X", /*->*/ "Out"); + ctx->SetOutputDim("Out", in_dim); + ctx->ShareLoD("X", /*->*/ "Out"); } }; class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: - SumOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + SumOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "the input tensors of sum operator.").AsDuplicable(); AddOutput("Out", "the output tensor of sum operator."); @@ -63,13 +59,16 @@ class SumGradOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - auto outputs = - ctx.MultiOutput(framework::GradVarName("X")); - auto dims = ctx.Input(framework::GradVarName("Out"))->dims(); - for (auto output : outputs) { - output->Resize(dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + auto out_grad_dims = ctx->GetInputDim(framework::GradVarName("Out")); + auto x_grad_names = ctx->Outputs(framework::GradVarName("X")); + size_t x_length = x_grad_names.size(); + std::vector x_grad_dims; + x_grad_dims.reserve(x_length); + for (size_t i = 0; i < x_length; ++i) { + x_grad_dims.push_back(out_grad_dims); } + ctx->SetOutputsDim(framework::GradVarName("X"), x_grad_dims); } }; diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc index a6e43964e9825cd1ced9e7c1bc8d691422248fee..5f22bf1df8720b60aba7cd75896d88cd1ad77635 100644 --- a/paddle/operators/top_k_op.cc +++ b/paddle/operators/top_k_op.cc @@ -22,26 +22,26 @@ class TopkOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of TopkOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) of TopkOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Indices"), - "Output(Indices) of TopkOp should not be null."); + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of TopkOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of TopkOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Indices"), + "Output(Indices) of TopkOp should not be null."); - auto *input = ctx.Input("X"); - const int k = static_cast(ctx.Attr("k")); + auto input_dims = ctx->GetInputDim("X"); + const int k = static_cast(ctx->Attrs().Get("k")); PADDLE_ENFORCE_GE(k, 1, "k must >= 1"); - PADDLE_ENFORCE_GE(input->dims().size(), 1, "input must have >= 1d shape"); - PADDLE_ENFORCE_GE(input->dims()[input->dims().size() - 1], k, + PADDLE_ENFORCE_GE(input_dims.size(), 1, "input must have >= 1d shape"); + PADDLE_ENFORCE_GE(input_dims[input_dims.size() - 1], k, "input must have >= k columns"); - framework::DDim dims = input->dims(); + framework::DDim dims = input_dims; dims[dims.size() - 1] = k; - ctx.Output("Out")->Resize(dims); - ctx.Output("Indices")->Resize(dims); + ctx->SetOutputDim("Out", dims); + ctx->SetOutputDim("Indices", dims); } }; diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc index 017a05326e9b397185d7c3530891884b11784783..0672f9342dac00ecc3f358450a9a203327cbb51f 100644 --- a/paddle/operators/transpose_op.cc +++ b/paddle/operators/transpose_op.cc @@ -24,12 +24,11 @@ class TransposeOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), - "Output(Out) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - std::vector axis = ctx.Attr>("axis"); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + std::vector axis = ctx->Attrs().Get>("axis"); size_t x_rank = x_dims.size(); size_t axis_size = axis.size(); @@ -51,14 +50,14 @@ class TransposeOp : public framework::OperatorWithKernel { for (size_t i = 0; i < axis_size; i++) { out_dims[i] = x_dims[axis[i]]; } - ctx.Output("Out")->Resize(out_dims); + ctx->SetOutputDim("Out", out_dims); } }; class TransposeOpMaker : public framework::OpProtoAndCheckerMaker { public: - TransposeOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) + TransposeOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", @@ -79,7 +78,7 @@ For example: [3, 4, 5]]) >> axis = [1, 0] >> output = input.transpose(axis) - >> output + >> output array([[0, 3], [1, 4], [2, 5]]) @@ -94,14 +93,15 @@ class TransposeOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - auto x_dims = ctx.Input("X")->dims(); - auto *x_grad = ctx.Output(framework::GradVarName("X")); - - if (x_grad) x_grad->Resize(x_dims); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx->GetInputDim("X"); + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); + if (ctx->HasOutput(framework::GradVarName("X"))) { + ctx->SetOutputDim(framework::GradVarName("X"), x_dims); + } } }; diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index 17ea48361bc597ccfeb80884d51900e6567aa057..2771df56086ff261728af84edcdf01cda3e45e9f 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -23,18 +23,18 @@ namespace operators { template class CPUUniformRandomKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& context) const override { - auto* tensor = context.Output("Out"); - T* data = tensor->mutable_data(context.GetPlace()); - unsigned int seed = static_cast(context.Attr("seed")); + void Compute(const framework::ExecutionContext& ctx) const override { + auto* tensor = ctx.Output("Out"); + T* data = tensor->mutable_data(ctx.GetPlace()); + unsigned int seed = static_cast(ctx.Attr("seed")); std::minstd_rand engine; if (seed == 0) { seed = std::random_device()(); } engine.seed(seed); std::uniform_real_distribution dist( - static_cast(context.Attr("min")), - static_cast(context.Attr("max"))); + static_cast(ctx.Attr("min")), + static_cast(ctx.Attr("max"))); int64_t size = tensor->numel(); for (int64_t i = 0; i < size; ++i) { data[i] = dist(engine); @@ -47,21 +47,20 @@ class UniformRandomOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of UniformRandomOp should not be null."); + void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of UniformRandomOp should not be null."); - PADDLE_ENFORCE(Attr("min") < Attr("max"), - "uniform_random's min must less then max"); - auto* tensor = ctx.Output("Out"); + PADDLE_ENFORCE( + ctx->Attrs().Get("min") < ctx->Attrs().Get("max"), + "uniform_random's min must less then max"); auto dims = Attr>("dims"); std::vector temp; temp.reserve(dims.size()); for (auto dim : dims) { temp.push_back(static_cast(dim)); } - tensor->Resize(framework::make_ddim(temp)); + ctx->SetOutputDim("Out", framework::make_ddim(temp)); } }; diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h index df5f71ed760952ed042d7ffa40a4319a73fb93bf..b523ef03c0053622bfda5b4bf07515c1b480b4af 100644 --- a/paddle/platform/enforce.h +++ b/paddle/platform/enforce.h @@ -107,7 +107,7 @@ struct EnforceNotMet : public std::exception { template inline typename std::enable_if::type throw_on_error( - int stat, const Args&... args) { + bool stat, const Args&... args) { if (UNLIKELY(!(stat))) { throw std::runtime_error(string::Sprintf(args...)); } diff --git a/paddle/pybind/.clang-format b/paddle/pybind/.clang-format new file mode 120000 index 0000000000000000000000000000000000000000..7d28cb3924707d39dafe20f4664fb17b5538996c --- /dev/null +++ b/paddle/pybind/.clang-format @@ -0,0 +1 @@ +../framework/.clang-format \ No newline at end of file diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt index 4f05406c7f74113d8fb10aa6914166e553858338..18ecbd1aa34c82d63ae7f8ec1bd8f81b35eee30b 100644 --- a/paddle/pybind/CMakeLists.txt +++ b/paddle/pybind/CMakeLists.txt @@ -1,6 +1,6 @@ if(WITH_PYTHON) cc_library(paddle_pybind SHARED - SRCS pybind.cc - DEPS pybind python backward + SRCS pybind.cc exception.cc protobuf.cc + DEPS pybind python backward proto_desc ${GLOB_OP_LIB}) endif(WITH_PYTHON) diff --git a/paddle/pybind/exception.cc b/paddle/pybind/exception.cc new file mode 100644 index 0000000000000000000000000000000000000000..ff79b12ee4b28c53ee04f4c170b5bca9ca28d14a --- /dev/null +++ b/paddle/pybind/exception.cc @@ -0,0 +1,34 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/pybind/exception.h" + +namespace paddle { +namespace pybind { + +void BindException(pybind11::module& m) { + static pybind11::exception exc(m, "EnforceNotMet"); + pybind11::register_exception_translator([](std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (const platform::EnforceNotMet& e) { + exc(e.what()); + } + }); + + m.def("__unittest_throw_exception__", [] { PADDLE_THROW("test exception"); }); +} + +} // namespace pybind +} // namespace paddle diff --git a/paddle/pybind/exception.h b/paddle/pybind/exception.h new file mode 100644 index 0000000000000000000000000000000000000000..70beac146046f74e23f747bab130483901a7d443 --- /dev/null +++ b/paddle/pybind/exception.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include "paddle/platform/enforce.h" +#include "pybind11/pybind11.h" +namespace paddle { +namespace pybind { + +extern void BindException(pybind11::module& m); +} // namespace pybind +} // namespace paddle diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc new file mode 100644 index 0000000000000000000000000000000000000000..218821b35bb6947181fedc56e002ad0285f6307d --- /dev/null +++ b/paddle/pybind/protobuf.cc @@ -0,0 +1,206 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/pybind/protobuf.h" +#include +#include +#include "paddle/framework/block_desc.h" +#include "paddle/framework/op_desc.h" +#include "paddle/framework/program_desc.h" +#include "paddle/framework/var_desc.h" + +// Cast boost::variant for PyBind. +// Copy from +// https://github.com/pybind/pybind11/issues/576#issuecomment-269563199 +namespace pybind11 { +namespace detail { + +// Can be replaced by a generic lambda in C++14 +struct variant_caster_visitor : public boost::static_visitor { + return_value_policy policy; + handle parent; + + variant_caster_visitor(return_value_policy policy, handle parent) + : policy(policy), parent(parent) {} + + template + handle operator()(T const &src) const { + return make_caster::cast(src, policy, parent); + } +}; + +template +struct variant_caster; + +template