diff --git a/.travis.yml b/.travis.yml index d0e2696f100e55f320e410afd6a3038db647f76f..c51e02eb79a9e53a2b8d1d663e8f0c3e0d8c3a61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ addons: - automake - libtool - ccache + ssh_known_hosts: 52.76.173.135 before_install: - if [[ "$JOB" == "check_style" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python @@ -42,6 +43,14 @@ script: - | timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi; + - | + if [[ "$JOB" != "build_doc" ]]; then exit 0; fi; + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi; + if [[ "$TRAVIS_BRANCH" != "develop" && ! "$TRAVIS_BRANCH" =~ ^v[[:digit:]]+\.[[:digit:]]+(\.[[:digit:]]+)?(-\S*)?$ ]]; then exit 0; fi; + export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/master/scripts/deploy/deploy_docs.sh + export DOCS_DIR=`pwd` + cd .. + curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH $DOCS_DIR $DOCS_DIR/build/doc notifications: email: on_success: change diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d4bb973ae87bb45ef4386a63c26ed62602f2cee..a60453ff4e3bba6e6cb3b3de915dd69afd3a1ec3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1,157 @@ -./doc/howto/dev/contribute_to_paddle_en.md +# Contribute Code + +We sincerely appreciate your contribution. This document explains our workflow and work style. + +## Workflow + +PaddlePaddle uses this [Git branching model](http://nvie.com/posts/a-successful-git-branching-model/). The following steps guide usual contributions. + +1. Fork + + Our development community has been growing fastly; it doesn't make sense for everyone to write into the official repo. So, please file Pull Requests from your fork. To make a fork, just head over to the GitHub page and click the ["Fork" button](https://help.github.com/articles/fork-a-repo/). + +1. Clone + + To make a copy of your fork to your local computers, please run + + ```bash + git clone https://github.com/your-github-account/paddle + cd paddle + ``` + +1. Create the local feature branch + + For daily works like adding a new feature or fixing a bug, please open your feature branch before coding: + + ```bash + git checkout -b my-cool-stuff + ``` + +1. Commit + + Before issuing your first `git commit` command, please install [`pre-commit`](http://pre-commit.com/) by running the following commands: + + ```bash + pip install pre-commit + pre-commit install + ``` + + Our pre-commit configuration requires clang-format 3.8 for auto-formating C/C++ code and yapf for Python. + + Once installed, `pre-commit` checks the style of code and documentation in every commit. We will see something like the following when you run `git commit`: + + ``` + ➜ git commit + CRLF end-lines remover...............................(no files to check)Skipped + yapf.................................................(no files to check)Skipped + Check for added large files..............................................Passed + Check for merge conflicts................................................Passed + Check for broken symlinks................................................Passed + Detect Private Key...................................(no files to check)Skipped + Fix End of Files.....................................(no files to check)Skipped + clang-formater.......................................(no files to check)Skipped + [my-cool-stuff c703c041] add test file + 1 file changed, 0 insertions(+), 0 deletions(-) + create mode 100644 233 + ``` + +1. Build and test + + Users can build PaddlePaddle natively on Linux and Mac OS X. But to unify the building environment and to make it easy for debugging, the recommended way is [using Docker](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/howto/dev/build_en.md). + +1. Keep pulling + + An experienced Git user pulls from the official repo often -- daily or even hourly, so they notice conflicts with others work early, and it's easier to resolve smaller conflicts. + + ```bash + git remote add upstream https://github.com/PaddlePaddle/Paddle + git pull upstream develop + ``` + +1. Push and file a pull request + + You can "push" your local work into your forked repo: + + ```bash + git push origin my-cool-stuff + ``` + + The push allows you to create a pull request, requesting owners of this [official repo](https://github.com/PaddlePaddle/Paddle) to pull your change into the official one. + + To create a pull request, please follow [these steps](https://help.github.com/articles/creating-a-pull-request/). + + If your change is for fixing an issue, please write ["Fixes "](https://help.github.com/articles/closing-issues-using-keywords/) in the description section of your pull request. Github would close the issue when the owners merge your pull request. + + Please remember to specify some reviewers for your pull request. If you don't know who are the right ones, please follow Github's recommendation. + + +1. Delete local and remote branches + + To keep your local workspace and your fork clean, you might want to remove merged branches: + + ```bash + git push origin :my-cool-stuff + git checkout develop + git pull upstream develop + git branch -d my-cool-stuff + ``` + +### Code Review + +- Please feel free to ping your reviewers by sending them the URL of your pull request via IM or email. Please do this after your pull request passes the CI. + +- Please answer reviewers' every comment. If you are to follow the comment, please write "Done"; please give a reason otherwise. + +- If you don't want your reviewers to get overwhelmed by email notifications, you might reply their comments by [in a batch](https://help.github.com/articles/reviewing-proposed-changes-in-a-pull-request/). + +- Reduce the unnecessary commits. Some developers commit often. It is recommended to append a sequence of small changes into one commit by running `git commit --amend` instead of `git commit`. + + +## Coding Standard + +### Code Style + +Our C/C++ code follows the [Google style guide](http://google.github.io/styleguide/cppguide.html). + +Our Python code follows the [PEP8 style guide](https://www.python.org/dev/peps/pep-0008/). + +Our build process helps to check the code style. In [`build.sh`](https://github.com/PaddlePaddle/Paddle/blob/b84e8226514b8bb4405c3c28e54aa5077193d179/paddle/scripts/docker/build.sh#L42), the entry point of our [builder Docker image](https://github.com/PaddlePaddle/Paddle/blob/b84e8226514b8bb4405c3c28e54aa5077193d179/Dockerfile#L88), the CMake argument `WITH_STYLE_CHECK` is set to `ON` by default. This flag is on + +Please install pre-commit, which automatically reformat the changes to C/C++ and Python code whenever we run `git commit`. To check the whole codebase, we can run the command `pre-commit run -a`, as in the [`check_style.sh` file](https://github.com/PaddlePaddle/Paddle/blob/b84e8226514b8bb4405c3c28e54aa5077193d179/paddle/scripts/travis/check_style.sh#L30), which is invoked by [our Travis CI configuration](https://github.com/PaddlePaddle/Paddle/blob/b84e8226514b8bb4405c3c28e54aa5077193d179/.travis.yml#L43). + +### Unit Tests + +Please remember to add related unit tests. + +- For C/C++ code, please follow [`google-test` Primer](https://github.com/google/googletest/blob/master/googletest/docs/Primer.md). + +- For Python code, please use [Python's standard `unittest` package](http://pythontesting.net/framework/unittest/unittest-introduction/). + + +### Writing Logs + +We use [glog](https://github.com/google/glog) for logging in our C/C++ code. + +For general information, please use `LOG`. For debug information, please use [`VLOG`](http://htmlpreview.github.io/?https://github.com/google/glog/blob/master/doc/glog.html#verbose). The reason is at [here](https://groups.google.com/a/chromium.org/d/msg/chromium-dev/3NDNd1KzXeY/AZKMMx37fdQJ). + +`VLOG` requires a *verbose level* parameter. For example: + +```c++ +VLOG(3) << "Operator FC is taking " << num_inputs << "inputs." +``` + +When we run a PaddlePaddle application or test, we can specify a verbose threshold. For example: + +```bash +GLOG_vmodule=buddy_allocator=2 \ +GLOG_v=10 \ +python \ +../python/paddle/v2/framework/tests/test_recurrent_op.py +``` + +This will enable VLOG messages generated by `buddy_allocator.{h,cc}` and in the verbose range of 0 to 3, so you will see above example VLOG message, which is in level 3. This suggests that we output overall messages in lower verbose levels, so they display with higher probability. When coding C++, please follow the verbose level convention as follows: + +- verbose level 1: [framework](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/framework) +- verbose level 3: [operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators) +- verbose level 5: [memory](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/memory), [platform](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/platform) +- verbose level 7: [math](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/math) diff --git a/benchmark/IntelOptimizedPaddle.md b/benchmark/IntelOptimizedPaddle.md new file mode 100644 index 0000000000000000000000000000000000000000..040f5ffa41968cbf93a817faa1db86c18956341e --- /dev/null +++ b/benchmark/IntelOptimizedPaddle.md @@ -0,0 +1,48 @@ +# Benchmark + +Machine: + +- Server + - Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz, 2 Sockets, 20 Cores per socket +- Laptop + - DELL XPS15-9560-R1745: i7-7700HQ 8G 256GSSD + - i5 MacBook Pro (Retina, 13-inch, Early 2015) +- Desktop + - i7-6700k + +System: CentOS release 6.3 (Final), Docker 1.12.1. + +PaddlePaddle: paddlepaddle/paddle:latest (TODO: will rerun after 0.11.0) + +- MKL-DNN tag v0.10 +- MKLML 2018.0.20170720 +- OpenBLAS v0.2.20 + +On each machine, we will test and compare the performance of training on single node using MKL-DNN / MKLML / OpenBLAS respectively. + +## Benchmark Model + +### Server +Test on batch size 64, 128, 256 on Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz + +Input image size - 3 * 224 * 224, Time: images/second + +- VGG-19 + +| BatchSize | 64 | 128 | 256 | +|--------------|-------| -----| --------| +| OpenBLAS | 7.82 | 8.62 | 10.34 | +| MKLML | 11.02 | 12.86 | 15.33 | +| MKL-DNN | 27.69 | 28.8 | 29.27 | + + +chart on batch size 128 +TBD + + - ResNet + - GoogLeNet + +### Laptop +TBD +### Desktop +TBD diff --git a/doc/design/images/asgd.gif b/doc/design/images/asgd.gif new file mode 100644 index 0000000000000000000000000000000000000000..4a0da7bf6df9326a2aab1638b77c5455c18b8c4e Binary files /dev/null and b/doc/design/images/asgd.gif differ diff --git a/doc/design/images/theta_star.gif b/doc/design/images/theta_star.gif new file mode 100644 index 0000000000000000000000000000000000000000..dd24d33e124396be3fc410c9b12f33148f64efe2 Binary files /dev/null and b/doc/design/images/theta_star.gif differ diff --git a/doc/design/parameter_average.md b/doc/design/parameter_average.md new file mode 100644 index 0000000000000000000000000000000000000000..2c4edee9fe31d502ea62b9fe5c8757c0a4c5e79f --- /dev/null +++ b/doc/design/parameter_average.md @@ -0,0 +1,72 @@ +# Averaging Parameter in PaddlePaddle + +## Why Averaging +In a large scale machine learning setup where the size of the training data is huge, it could take us a large number of iterations over the training data before we can achieve the optimal values of parameters of our model. Looking at the problem setup, it is desirable if we can obtain the optimal values of parameters by going through the data in as few passes as we can. + +Polyak and Juditsky (1992) showed that the test performance of simple average of parameters obtained by Stochastic Gradient Descent (SGD) is as good as that of parameter values that are obtained by training the model over and over again, over the training dataset. + +Hence, to accelerate the speed of Stochastic Gradient Descent, Averaged Stochastic Gradient Descent (ASGD) was proposed in Polyak and Juditsky (1992). For ASGD, the running average of parameters obtained by SGD, is used as the estimator for
. The averaging is done as follows: + +
+ +We propose averaging for any optimizer similar to how ASGD performs it, as mentioned above. + +### How to perform Parameter Averaging in PaddlePaddle + +Parameter Averaging in PaddlePaddle works in the following way during training : +1. It will take in an instance of a normal optimizer as an input, e.g. RMSPropOptimizer +2. The optimizer itself is responsible for updating the parameters. +3. The ParameterAverageOptimizer maintains a separate copy of the parameters for itself: + 1. In concept, the values of this copy are the average of the values of the parameters in the most recent N batches. + 2. However, saving all the N instances of the parameters in memory is not feasible. + 3. Therefore, an approximation algorithm is used. + +Hence, overall we have have two copies of the parameters: one for the optimizer itself, and one for the ParameterAverageOptimizer. The former should be used in back propagation, while the latter should be used during testing and should be saved. + +During the testing/ saving the model phase, we perform the following steps: +1. Perform the delayed operations. +2. Save current values of the parameters to a temporary variable. +3. Replace the values of the parameters with the averaged values. +4. Perform testing and/or save the parameters. +5. Restore the values of the parameters once done. + +### How to implement Averaging of Parameter in PaddlePaddle + +We can add the ParameterAverageOptimizer op to the graph through Python API. Using this approach, we manually add this op to the graph and direct the output of the optimizer op to this op during training. + + **Advantages**: + - Allows for greater flexibility to the users of PaddlePaddle. Using this approach, the users can plug different optimizers into ParameterAverageOptimizer by passing in the optimizer to the op. + - Makes it easy for the users to customize and extend the framework. + + **Disadvantages**: + - Implementation requires re-writing the averaging methodology in Python. + +### Low-Level implementation + +In the new design, we propose to create a new operation for averaging parameter updates (ParameterAverageOptimizer). For now, we can add an op that takes in the following as input: +- the optimizer +- the window_size to keep the updates + +The ParameterAverageOptimizer op can be like any other operator with its own CPU/GPU implementation either using Eigen or separate CPU and GPU kernels. As the initial implementation, we can implement the kernel using Eigen following the abstraction pattern implemented for [Operators](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/rmsprop_op.h). We also want to support the case when the Trainer/Optimizer runs on the GPU while ParameterAverageOptimizer runs on a CPU. + +The idea of building an op for averaging is in sync with the refactored PaddlePaddle philosophy of using operators to represent any computation unit. The way the op will be added to the computation graph will be decided by the [layer functions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md#layer-function) in Python API. + +### Python API implementation for ParameterAverageOptimizer + +Based on Polyak and Juditsky (1992), we can generalize the averaging of updates to any optimizer. The input to the op would be the following: +- Any optimizer (RMSProp , AdaGrad etc.) +- A window size. The op keeps accumulating updated parameter values over a window of N batches and takes an average. Move the averaged value to a buffer when window is full to avoid loss of precision. + +Using the ParameterAverageOptimizer op, any user can add the operation to their computation graphs. However, this will require a lot of lines of code and we should design Python APIs that support averaging. As per the PaddlePaddle [Python API design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md), the layer functions are responsible for creating operators, operator parameters and variables. Since ParameterAverageOptimizer will be an operator, it makes sense to create it in the layer functions. +We will have a wrapper written in Python that will support the functionality and implement the actual core computation in C++ core as we have done for other [Optimizers](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/rmsprop_op.cc) + +#### Creation of the ParameterAverageOptimizer operator +There are two ways for creating the ParameterAverageOptimizer op: +1. We create the op immediately while building the computation graph. +2. We add the op in a lazy manner, just before the backward pass, similar to the way the optimization ops are added. + +The proposal is to add the op immediately while building the computation graph. + +#### High-level API + +In PaddlePaddle Python API, users will primarily rely on [layer functions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md#layer-function) to create neural network layers. Hence, we also need to provide parameter average functionality in layer functions. diff --git a/doc/faq/parameter/index_cn.rst b/doc/faq/parameter/index_cn.rst index c721b623183cc7d8d17e2c9fb1635ea07b8970cc..6fa0c64413be1616a435640b0347904a49873349 100644 --- a/doc/faq/parameter/index_cn.rst +++ b/doc/faq/parameter/index_cn.rst @@ -75,7 +75,7 @@ PaddlePaddle目前支持8种learning_rate_schedule，这8种learning_rate_schedu optimizer = paddle.optimizer.Adam( learning_rate=1e-3, - learning_rate_schedule="manual", + learning_rate_schedule="pass_manual", learning_rate_args="1:1.0,2:0.9,3:0.8",) 在该示例中，当已训练pass数小于等于1时，学习率为 :code:`1e-3 * 1.0`；当已训练pass数大于1小于等于2时，学习率为 :code:`1e-3 * 0.9`；当已训练pass数大于2时，学习率为 :code:`1e-3 * 0.8`。 diff --git a/doc/howto/cross_compiling/cross_compiling_for_android.md b/doc/howto/cross_compiling/cross_compiling_for_android.md new file mode 100644 index 0000000000000000000000000000000000000000..161863e5c0a2c002af7d7611dad53c2c19148722 --- /dev/null +++ b/doc/howto/cross_compiling/cross_compiling_for_android.md @@ -0,0 +1,153 @@ +# Build PaddlePaddle for Android + +There are two approaches to build PaddlePaddle for Android: using Docker and on Linux without Docker. + +## Cross-Compiling Using Docker + +Docker-based cross-compiling is the recommended approach because Docker runs on all major operating systems, including Linux, Mac OS X, and Windows. + +### Build the Docker Image + +The following steps pack all the tools that we need to build PaddlePaddle into a Docker image. + +```bash +$ git clone https://github.com/PaddlePaddle/Paddle.git +$ cd Paddle +$ docker build -t paddle:dev-android . -f Dockerfile.android +``` + +### Build the Inference Library + +We can run the Docker image we just created to build the inference library of PaddlePaddle for Android using the command below: + +```bash +$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" paddle:dev-android +``` + +The Docker image accepts two arguments `ANDROID_ABI` and `ANDROID_API`: + +| Argument | Optional Values | Default | +|-----------------|-------------------------|---------| +|`ANDROID_ABI` |`armeabi-v7a, arm64-v8a` | `armeabi-v7a` | +|`ANDROID_API` |`>= 21` | `21` | + +The ARM-64 architecture (`arm64-v8a`) requires at least level 21 of Android API. + +The default entry-point of the Docker image, [`paddle/scripts/docker/build_android.sh`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh) generates the [Android cross-compiling standalone toolchain](https://developer.android.com/ndk/guides/standalone_toolchain.html) based on the argument: `ANDROID_ABI` or `ANDROID_API`. For information about other configuration arguments, please continue reading. + +The above command generates and outputs the inference library in `$PWD/install_android` and puts third-party libraries in `$PWD/install_android/third_party`. + +## Cross-Compiling on Linux + +The Linux-base approach to cross-compile is to run steps in `Dockerfile.android` manually on a Linux x64 computer. + +### Setup the Environment + +To build for Android's, we need [Android NDK]( +https://developer.android.com/ndk/downloads/index.html): + +```bash +wget -q https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip +unzip -q android-ndk-r14b-linux-x86_64.zip +``` + +Android NDK includes everything we need to build the [*standalone toolchain*](https://developer.android.com/ndk/guides/standalone_toolchain.html), which in then used to build PaddlePaddle for Android. (We plan to remove the intermediate stage of building the standalone toolchain in the near future.) + +- To build the standalone toolchain for `armeabi-v7a` and Android API level 21: + + ```bash + your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain.sh \ + --arch=arm --platform=android-21 --install-dir=your/path/to/arm_standalone_toolchain + ``` + + The generated standalone toolchain will be in `your/path/to/arm_standalone_toolchain`. + +- To build the standalone toolchain for `arm64-v8a` and Android API level 21: + + ```bash + your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain.sh \ + --arch=arm64 --platform=android-21 --install-dir=your/path/to/arm64_standalone_toolchain + ``` + + The generated standalone toolchain will be in `your/path/to/arm64_standalone_toolchain`. + +**Please be aware that the minimum level of Android API required by PaddlePaddle is 21.** + +### Cross-Compiling Arguments + +CMake supports [choosing the toolchain](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling). PaddlePaddle provides [`android.cmake`](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/android.cmake), which configures the Android cross-compiling toolchain for CMake. `android.cmake` is not required for CMake >= 3.7, which support Android cross-compiling. PaddlePaddle detects the CMake version, for those newer than 3.7, it uses [the official version](https://cmake.org/cmake/help/v3.7/manual/cmake-toolchains.7.html#cross-compiling). + +Some other CMake arguments you need to know: + +- `CMAKE_SYSTEM_NAME` must be `Android`. This tells PaddlePaddle's CMake system to cross-compile third-party dependencies. This also changes some other CMake arguments like `WITH_GPU=OFF`, `WITH_AVX=OFF`, `WITH_PYTHON=OFF`, and `WITH_RDMA=OFF`. +- `WITH_C_API` must be `ON`, to build the C-based inference library for Android. +- `WITH_SWIG_PY` must be `OFF` because the Android platform doesn't support SWIG-based API. + +Some Android-specific arguments: + +- `ANDROID_STANDALONE_TOOLCHAIN`: the absolute path of the Android standalone toolchain, or the path relative to the CMake build directory. PaddlePaddle's CMake extensions would derive the cross-compiler, sysroot and Android API level from this argument. +- `ANDROID_TOOLCHAIN`: could be `gcc` or `clang`. The default value is `clang`. + - For CMake >= 3.7, it should anyway be `clang`. For older versions, it could be `gcc`. + - Android's official `clang` requires `glibc` >= 2.15. +- `ANDROID_ABI`: could be `armeabi-v7a` or `arm64-v8a`. The default value is `armeabi-v7a`. +- `ANDROID_NATIVE_API_LEVEL`: could be derived from the value of `ANDROID_STANDALONE_TOOLCHAIN`. +- `ANROID_ARM_MODE`: + - could be `ON` or `OFF`, and defaults to `ON`, when `ANDROID_ABI=armeabi-v7a`; + - no need to specify when `ANDROID_ABI=arm64-v8a`. +- `ANDROID_ARM_NEON`: indicates if to use NEON instructions. + - could be `ON` or `OFF`, and defaults to `ON`, when `ANDROID_ABI=armeabi-v7a`; + - no need to specify when `ANDROID_ABI=arm64-v8a`. + +Other useful arguments: + +- `USE_EIGEN_FOR_BLAS`: indicates if using Eigen. Could be `ON` or `OFF`, defaults to `OFF`. +- `HOST_C/CXX_COMPILER`: specifies the host compiler, which is used to build the host-specific protoc and target-specific OpenBLAS. It defaults to the value of the environment variable `CC`, or `cc`. + +Some frequent configurations for your reference: + +```bash +cmake -DCMAKE_SYSTEM_NAME=Android \ + -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm_standalone_toolchain \ + -DANDROID_ABI=armeabi-v7a \ + -DANDROID_ARM_NEON=ON \ + -DANDROID_ARM_MODE=ON \ + -DUSE_EIGEN_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +``` +cmake -DCMAKE_SYSTEM_NAME=Android \ + -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm64_standalone_toolchain \ + -DANDROID_ABI=arm64-v8a \ + -DUSE_EIGEN_FOR_BLAS=OFF \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_SWIG_PY=OFF \ + .. +``` + + +There are some other arguments you might want to configure. + +- `CMAKE_BUILD_TYPE=MinSizeRel` minimizes the size of library. +- `CMAKE_BUILD_TYPE-Release` optimizes the runtime performance. + +Our own tip for performance optimization to use clang and Eigen or OpenBLAS: +- `CMAKE_BUILD_TYPE=Release` +- `ANDROID_TOOLCHAIN=clang` +- `USE_EIGEN_BLAS=ON` for `armeabi-v7a`, or `USE_EIGEN_FOR_BLAS=OFF` for `arm64-v8a`. + +### Build and Install + +After running `cmake`, we can run `make; make install` to build and install. + +Before building, you might want to remove the `third_party` and `build` directories including pre-built libraries for other architectures. + +After building，in the directory `CMAKE_INSTALL_PREFIX`, you will find three sub-directories: + +- `include`: the header file of the inference library, +- `lib`: the inference library built for various Android ABIs, +- `third_party`: dependent third-party libraries built for Android. diff --git a/doc/howto/cross_compiling/cross_compiling_for_android_cn.md b/doc/howto/cross_compiling/cross_compiling_for_android_cn.md index 1fc58c37cc9151d5e4d99b939e30c29aa99e04f1..58e4dd9c3fe43f963d00152aa4f456fadbb12bf3 100644 --- a/doc/howto/cross_compiling/cross_compiling_for_android_cn.md +++ b/doc/howto/cross_compiling/cross_compiling_for_android_cn.md @@ -1,7 +1,7 @@ # 构建Android平台上的PaddlePaddle库 用户可通过如下两种方式，交叉编译Android平台上适用的PaddlePaddle库： -- 基于Docker容器的编译方式 +- 基于Docker容器的编译方式 - 基于Linux交叉编译环境的编译方式 ## 基于Docker容器的编译方式 @@ -26,14 +26,14 @@ Android的Docker开发镜像向用户提供两个可配置的参数： |`ANDROID_API` |`>= 21` | `21` | - 编译`armeabi-v7a`，`Android API 21`的PaddlePaddle库 -```bash -$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" username/paddle-android:dev -``` + ```bash + $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" username/paddle-android:dev + ``` -- 编译`arm64-v8a`，`Android API 21`的PaddlePaddle库 -```bash -$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=arm64-v8a" -e "ANDROID_API=21" username/paddle-android:dev -``` +- 编译`arm64-v8a`，`Android API 21`的PaddlePaddle库 + ```bash + $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=arm64-v8a" -e "ANDROID_API=21" username/paddle-android:dev + ``` 执行上述`docker run`命令时，容器默认执行[paddle/scripts/docker/build_android.sh](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh)脚本。该脚本中记录了交叉编译Android版PaddlePaddle库常用的CMake配置，并且会根据`ANDROID_ABI`和`ANDROID_API`自动构建独立工具链、进行编译和安装。由于arm64架构要求Android API不小于21。因此当`ANDROID_ABI=arm64-v8a`，`ANDROID_API<21`时，Docker容器中将默认使用`Android API 21`的编译工具链。用户可以参考下文**配置交叉编译参数**章节，根据个人的需求修改定制Docker容器所执行的脚本。编译安装结束之后，PaddlePaddle的C-API库将被安装到`$PWD/install_android`目录，所依赖的第三方库同时也被安装到`$PWD/install_android/third_party`目录。 @@ -82,16 +82,16 @@ CMake系统对交叉编译提供了支持[cmake-toolchains](https://cmake.org/cm Android平台可选配置参数： - `ANDROID_STANDALONE_TOOLCHAIN`，独立工具链所在的绝对路径，或者相对于构建目录的相对路径。PaddlePaddle的CMake系统将根据该值自动推导和设置需要使用的交叉编译器、sysroot、以及Android API级别；否则，用户需要在cmake时手动设置这些值。无默认值。 -- `ANDROID_TOOLCHAIN`，目标工具链。可设置`gcc/clang`，默认值为`clang`。 - - CMake 3.7以上，将会始终使用`clang`工具链；CMake 3.7以下，可设置`ANDROID_TOOLCHAIN=gcc`以使用`gcc`工具链。 +- `ANDROID_TOOLCHAIN`，目标工具链。可设置`gcc/clang`，默认值为`clang`。 + - CMake 3.7以上，将会始终使用`clang`工具链；CMake 3.7以下，可设置`ANDROID_TOOLCHAIN=gcc`以使用`gcc`工具链。 - Android官方提供的`clang`编译器要求系统支持`GLIBC 2.15`以上。 - `ANDROID_ABI`，目标架构ABI。目前支持`armeabi-v7a`和`arm64-v8a`，默认值为`armeabi-v7a`。 - `ANDROID_NATIVE_API_LEVEL`，工具链的Android API级别。若没有显式设置，PaddlePaddle将根据`ANDROID_STANDALONE_TOOLCHAIN`的值自动推导得到。 -- `ANROID_ARM_MODE`，是否使用ARM模式。 - - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； +- `ANROID_ARM_MODE`，是否使用ARM模式。 + - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； - `ANDROID_ABI=arm64-v8a`时，不需要设置。 -- `ANDROID_ARM_NEON`，是否使用NEON指令。 - - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； +- `ANDROID_ARM_NEON`，是否使用NEON指令。 + - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； - `ANDROID_ABI=arm64-v8a`时，不需要设置。 其他配置参数： @@ -119,7 +119,7 @@ cmake -DCMAKE_SYSTEM_NAME=Android \ -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm64_standalone_toolchain \ -DANDROID_ABI=arm64-v8a \ -DUSE_EIGEN_FOR_BLAS=OFF \ - -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ -DWITH_C_API=ON \ -DWITH_SWIG_PY=OFF \ .. @@ -128,8 +128,8 @@ cmake -DCMAKE_SYSTEM_NAME=Android \ 用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS_MINSIZEREL/RELEASE`来影响PaddlePaddle的编译过程。 **性能TIPS**，为了达到最快的计算速度，在CMake参数配置上，有以下建议： -- 设置`CMAKE_BUILD_TYPE`为`Release` -- 使用`clang`编译工具链 +- 设置`CMAKE_BUILD_TYPE`为`Release` +- 使用`clang`编译工具链 - `armeabi-v7a`时，设置`USE_EIGEN_BLAS=ON`，使用Eigen进行矩阵计算；`arm64-v8a`时，设置`USE_EIGEN_FOR_BLAS=OFF`，使用OpenBLAS进行矩阵计算 ### 编译和安装 diff --git a/doc/howto/cross_compiling/cross_compiling_for_ios_cn.md b/doc/howto/cross_compiling/cross_compiling_for_ios_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..32c490d9aa4202e17aa1784a45a317c5307b98ea --- /dev/null +++ b/doc/howto/cross_compiling/cross_compiling_for_ios_cn.md @@ -0,0 +1,99 @@ +# 构建iOS平台上的PaddlePaddle库 +交叉编译iOS平台上适用的PaddlePaddle库，需要在MacOS系统上进行。本文的将介绍在MacOS上，从源码交叉编译iOS平台上适用的PaddlePaddle库。 + +## 准备交叉编译环境 +Apple官方为iOS开发提供了完整的交叉编译工具和集成开发环境，用户从App Store下载安装Xcode即可。也可自行前往官网下载，[Xcode](https://developer.apple.com/cn/xcode/)。安装完成之后，可在命令行执行`xcodebuild -version`，判断是否安装成功。 + +```bash +$ xcodebuild -version +Xcode 9.0 +Build version 9A235 +``` + +## 配置交叉编译参数 + +PaddlePaddle为交叉编译提供了工具链配置文档[cmake/cross_compiling/ios.cmake](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/ios.cmake)，以提供一些默认的编译器和编译参数配置。 + +交叉编译iOS版本的PaddlePaddle库时，有一些必须配置的参数： + +- `CMAKE_SYSTEM_NAME`，CMake编译的目标平台，必须设置为`iOS`。在设置`CMAKE_SYSTEM_NAME=iOS`后，PaddlePaddle的CMake系统会自动编译所有的第三方依赖库，并且强制设置一些PaddlePaddle参数的值（`WITH_C_API=ON`、`WITH_GPU=OFF`、`WITH_AVX=OFF`、`WITH_PYTHON=OFF`、`WITH_RDMA=OFF`）。 +- `WITH_C_API`，是否编译C-API预测库，必须设置为ON。在iOS平台上只支持使用C-API来预测。 +- `WITH_SWIG_PY`，必须设置为ON。在iOS平台上不支持通过swig调用来训练或者预测。 + +iOS平台可选配置参数： + +- `IOS_PLATFORM`，可设置为`OS/SIMULATOR`，默认值为`OS`。 + - `OS`，构建目标为`arm`架构的iPhone或者iPad等物理设备。 + - `SIMULATOR`，构建目标为`x86`架构的模拟器平台。 +- `IOS_ARCH`，目标架构。针对不同的`IOS_PLATFORM`，可设置的目标架构如下表所示： + + | IOS_PLATFORM | IOS_ARCH | + |--------------|----------------------| + | OS | armv7, armv7s, arm64 (默认) | + | SIMULATOR | i386, x86_64 (默认) | + +- `IOS_DEPLOYMENT_TARGET`，最小的iOS部署版本，默认值为`7.0`。 +- `IOS_ENABLE_BITCODE`，是否使能[Bitcode](https://developer.apple.com/library/content/documentation/IDEs/Conceptual/AppDistributionGuide/AppThinning/AppThinning.html#//apple_ref/doc/uid/TP40012582-CH35-SW3)，可设置`ON/OFF`，默认值为`ON`。 +- `IOS_USE_VECLIB_FOR_BLAS`，是否使用[vecLib](https://developer.apple.com/documentation/accelerate/veclib)框架进行BLAS矩阵计算，可设置`ON/OFF`，默认值为`OFF`。 +- `IOS_DEVELOPMENT_ROOT`，`Developer`目录，可显式指定为`/path/to/platform/Developer`。若未显式指定，PaddlePaddle将会根据`IOS_PLATFORM`自动选择`Xcode`对应`platform`的`Developer`目录。 +- `IOS_SDK_ROOT`，所使用`SDK`的根目录，可显式指定为`/path/to/platform/Developer/SDKs/SDK`。若未显式指定，PaddlePaddle将会自动选择`IOS_DEVELOPMENT_ROOT`目录下最新的`SDK`版本。 + +其他配置参数： + +- `USE_EIGEN_FOR_BLAS`，是否使用Eigen库进行矩阵计算，在`IOS_USE_VECLIB_FOR_BLAS=OFF`时有效。可设置`ON/OFF`，默认值为`OFF`。 +- `HOST_C/CXX_COMPILER`，宿主机的C/C++编译器。默认值为环境变量`CC/CXX`的值；若环境变量`CC/CXX`未设置，则使用`cc/c++`编译器。 + +常用的cmake配置如下： + +```bash +cmake -DCMAKE_SYSTEM_NAME=iOS \ + -DIOS_PLATFORM=OS \ + -DIOS_ARCH="arm64" \ + -DIOS_ENABLE_BITCODE=ON \ + -DIOS_USE_VECLIB_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_TESTING=OFF \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +```bash +cmake -DCMAKE_SYSTEM_NAME=iOS \ + -DIOS_PLATFORM=SIMULATOR \ + -DIOS_ARCH="x86_64" \ + -DIOS_USE_VECLIB_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_TESTING=OFF \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +用户还可根据自己的需求设置其他编译参数。比如希望最小化生成库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望得到最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS`来影响PaddlePaddle的编译过程。 + +**性能TIPS**，为了达到最快的计算速度，在CMake参数配置上，有以下建议： + +- 设置`CMAKE_BUILD_TYPE`为`Release` +- 设置`IOS_USE_VECLIB_FOR_BLAS=ON`，调用`vecLib`框架提供的BLAS函数进行矩阵计算。 + +## 编译和安装 + +CMake配置完成后，执行以下命令，PaddlePaddle将自动下载和编译所有第三方依赖库、编译和安装PaddlePaddle预测库。 + +``` +$ make +$ make install +``` + +注意：如果你曾在源码目录下编译过其他平台的PaddlePaddle库，请先使用`rm -rf`命令删除`third_party`目录和`build`目录，以确保所有的第三方依赖库和PaddlePaddle代码都是针对新的CMake配置重新编译的。 + +执行完安装命令后，`your/path/to/install`目录中会包含以下内容： + +- `include`目录，其中包含所有C-API的头文件 +- `lib`目录，其中包含PaddlePaddle的C-API静态库 +- `third_party`目录，其中包含所依赖的所有第三方库 + +注意，不同架构的PaddlePaddle库建议安装到不同的目录下，然后使用`lipo`工具将多个静态库合并成一个支持多个架构的fat库。 + +自此，PaddlePaddle库已经安装完成，用户可将合成的fat库用于深度学习相关的iOS App中，调用方法见C-API文档。 diff --git a/doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md b/doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md index 085b5dda1615a9af918b59870db460fcc5acdcca..6e983645faaed1f67edaeeb82ddbef9cef6bb85f 100644 --- a/doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md +++ b/doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md @@ -1,39 +1,36 @@ # 构建Raspberry Pi平台上的PaddlePaddle库 -对于Rasspberry Pi系统，用户可通过ssh等方式登录到Raspberry Pi系统上，按照[源码编译PaddlePaddle](http://www.paddlepaddle.org/doc_cn/getstarted/build_and_install/cmake/build_from_source_cn.html)相关文档所述，直接编译Raspberry Pi平台上适用的PaddlePaddle库。 +通常有两个方法来构建基于 Rasspberry Pi 的版本： -用户也可以在自己熟悉的开发平台上，通过交叉编译的方式来编译。这篇文档将以Linux x86-64平台为例，介绍交叉编译Raspberry Pi平台上适用的PaddlePaddle的方法和步骤。 +1. 通过ssh等方式登录到Raspberry Pi系统上来构建。所需的开发工具和第三方库可以参考 [`/Dockerfile`](https://github.com/PaddlePaddle/Paddle/blob/develop/Dockerfile)。 -## 准备交叉编译环境 +1. 另一个方法是交叉编译。这篇文档介绍在 Linux/x64 上交叉编译Raspberry Pi平台上适用的PaddlePaddle的方法和步骤。 -从源码交叉编译PaddlePaddle，用户需要提前准备好交叉编译环境。用户可自行前往[github](https://github.com/raspberrypi/tools)下载Raspberry Pi平台使用的C/C++交叉编译工具链，也可通过以下命令获取： +## 安装交叉编译器 + +克隆下面 Github repo ```bash git clone https://github.com/raspberrypi/tools.git ``` -该github仓库中包含若干个预编译好的、针对不同平台的编译工具。宿主机是Linux x86-64环境，则需选用`arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64`下的作为编译工具，所使用的编译器为arm-linux-gnueabihf-gcc 4.8.3。 - -注意，该编译工具链需要系统glibc支持2.14以上。 +即可在 `./tools/tree/master/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64` 目录里找到交叉编译器 arm-linux-gnueabihf-gcc 4.8.3。运行该编译工具链需要一台 Linux x64 机器上以及 2.14版本以上的 glibc。 ## 配置交叉编译参数 -CMake系统对交叉编译提供了支持[cmake-toolchains](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling)。为了简化cmake配置，PaddlePaddle为交叉编译提供了工具链配置文档[cmake/cross_compiling/raspberry_pi.cmake](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/raspberry_pi.cmake)，以提供一些默认的编译器和编译参数相关配置。 +CMake[支持交叉编译](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling)。PaddlePaddle for Raspberry Pi的配置信息在[cmake/cross_compiling/raspberry_pi.cmake](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/raspberry_pi.cmake)。 交叉编译Raspberry Pi版本PaddlePaddle库时，有一些必须配置的参数： -- `CMAKE_SYSTEM_NAME`，CMake编译的目标平台，必须配置为`RPi`。在设置`CMAKE_SYSTEM_NAME=RPi`后，PaddlePaddle的CMake系统才认为在是在交叉编译Raspberry Pi系统的版本，并自动编译宿主机版protoc可执行文件、目标机版protobuf库、以及目标机版OpenBLAS库。 - -Raspberry Pi平台可选配置参数： +- `CMAKE_SYSTEM_NAME`：CMake编译的目标平台，必须配置为`RPi`。在设置`CMAKE_SYSTEM_NAME=RPi`后，PaddlePaddle的CMake系统才认为在是在交叉编译Raspberry Pi系统的版本，并自动编译宿主机版protoc可执行文件、目标机版protobuf库、以及目标机版OpenBLAS库。 -- `RPI_TOOLCHAIN`，编译工具链所在的绝对路径，或者相对于构建目录的相对路径。PaddlePaddle的CMake系统将根据该值自动设置需要使用的交叉编译器；否则，用户需要在cmake时手动设置这些值。无默认值。 -- `RPI_ARM_NEON`，是否使用NEON指令。目前必须设置成`ON`，默认值为`ON`。 +- `RPI_TOOLCHAIN`：编译工具链所在的绝对路径，或者相对于构建目录的相对路径。PaddlePaddle的CMake系统将根据该值自动设置需要使用的交叉编译器；否则，用户需要在cmake时手动设置这些值。无默认值。 -其他配置参数： +- `RPI_ARM_NEON`：是否使用NEON指令。目前必须设置成`ON`，默认值为`ON`。 - `HOST_C/CXX_COMPILER`，宿主机的C/C++编译器。在编译宿主机版protoc可执行文件和目标机版OpenBLAS库时需要用到。默认设置成环境变量`CC`的值；若环境变量`CC`没有设置，则设置成`cc`编译器。 -cmake参数如下； +一个常用的CMake配置如下： ``` cmake -DCMAKE_SYSTEM_NAME=RPi \ @@ -47,7 +44,9 @@ cmake -DCMAKE_SYSTEM_NAME=RPi \ .. ``` -用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS_MINSIZEREL/RELEASE`来影响PaddlePaddle的编译过程。 +其中`WITH_C_API=ON`表示需要构建推理库。 + +用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。 ## 编译和安装 @@ -60,6 +59,4 @@ make install 注意：如果你曾经在源码目录下编译过其他平台的PaddlePaddle库，请先使用`rm -rf`命令删除`third_party`目录和`build`目录，以确保所有的第三方依赖库和PaddlePaddle代码都是针对新的CMake配置重新编译的。 -执行完安装命令后，由于上一步cmake配置中`WITH_C_API`设置为`ON`，`your/path/to/install`目录中会包含`include`和`lib`目录，其中`include`中包含C-API的头文件，`lib`中包含一个Raspberry Pi版本的库。 - -更多的编译配置见[源码编译PaddlePaddle](http://www.paddlepaddle.org/doc_cn/getstarted/build_and_install/cmake/build_from_source_cn.html)相关文档。 +执行完安装命令后，`your/path/to/install`目录中会包含`include`和`lib`目录，其中`include`中包含C-API的头文件，`lib`中包含一个Raspberry Pi版本的库。 diff --git a/doc/howto/cross_compiling/cross_compiling_for_raspberry_en.md b/doc/howto/cross_compiling/cross_compiling_for_raspberry_en.md new file mode 100644 index 0000000000000000000000000000000000000000..3c1a5950ff9553bb725d5a96e3fdf2e5e9f6f95c --- /dev/null +++ b/doc/howto/cross_compiling/cross_compiling_for_raspberry_en.md @@ -0,0 +1,62 @@ +# Build PaddlePaddle for Raspberry Pi + +You may use any of the following two approaches to build the inference library of PaddlePaddle for Raspberry Pi: + +1. Build using SSH: Log in to a Raspberry Pi using SSH and build the library. The required development tools and third-party dependencies are listed in here: [`/Dockerfile`](https://github.com/PaddlePaddle/Paddle/blob/develop/Dockerfile). + +1. Cross-compile: We talk about how to cross-compile PaddlePaddle for Raspberry Pi on a Linux/x64 machine, in more detail in this article. + +## The Cross-Compiling Toolchain + +Step 1. Clone the Github repo by running the following command. + +```bash +git clone https://github.com/raspberrypi/tools.git +``` + +Step 2. Use the pre-built cross-compiler found in `./tools/tree/master/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64`. To run it on a Linux computer, glibc version >= 2.14 is needed. + +## CMake Arguments + +CMake supports [cross-compiling](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling). All CMake configuration arguments required for the cross-compilation for Raspberry Pi can be found in [`cmake/cross_compiling/raspberry_pi.cmake`](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/raspberry_pi.cmake). + +Some important arguments that need to be set: + +- `CMAKE_SYSTEM_NAME`: The target platform. Must be `RPi`. + +- `RPI_TOOLCHAIN`: The absolute path of the cross-compiling toolchain. + +- `RPI_ARM_NEON`: Use ARM NEON Intrinsics. This is a required argument and set default to `ON`. + +- `HOST_C/CXX_COMPILER`: The C/C++ compiler for the host. It is used to build building tools running on the host, for example, protoc. + +A commonly-used CMake configuration is as follows: + +``` +cmake -DCMAKE_SYSTEM_NAME=RPi \ + -DRPI_TOOLCHAIN=your/path/to/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64 \ + -DRPI_ARM_NEON=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_GPU=OFF \ + -DWITH_C_API=ON \ + -DWITH_PYTHON=OFF \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +To build the inference library, please set the argument WITH\_C\_API to ON: `WITH_C_API=ON`. + +You can add more arguments. For example, to minimize the size of the generated inference library, you may use `CMAKE_BUILD_TYPE=MinSizeRel`. For performance optimization, you may use `CMAKE_BUILD_TYPE=Release`. + +## Build and Install + +The following commands build the inference library of PaddlePaddle for Raspberry Pi and third-party dependencies. + +```bash +make +make install +``` + + The intermediate files will be stored in `build`. Third-party libraries will be located in `build/third_party`. If you have already built it for other platforms like Android or iOS, you may want to clear these directories by running the command: `rm -rf build`. + +The infernece library will be in `your/path/to/install/lib`, with related header files in `your/path/to/install/include`. diff --git a/doc/howto/dev/contribute_to_paddle_en.md b/doc/howto/dev/contribute_to_paddle_en.md deleted file mode 100644 index 40d1eb62d722244139cc84eb170c190d988f5626..0000000000000000000000000000000000000000 --- a/doc/howto/dev/contribute_to_paddle_en.md +++ /dev/null @@ -1,219 +0,0 @@ -# Contribute Code - -We sincerely appreciate your contributions. You can use fork and pull request -workflow to merge your code. - -## Code Requirements -- Your code comments must be fully documented by - [Doxygen](http://www.stack.nl/~dimitri/doxygen/) style. -- Make sure the compiler option `WITH_STYLE_CHECK` is on and the compiler - passes the code style check. -- All code must have unit test. -- Pass all unit tests. - -The following tutorial guides you into submitting your contibution. - -## [Creating a Fork](https://help.github.com/articles/fork-a-repo/) - -Just head over to the GitHub page and click the "Fork" button. -It's just that simple. - -## Clone - -Clone remote repository. - -```bash -➜ git clone https://github.com/USERNAME/Paddle -➜ cd Paddle -``` - -## Create a local branch - -Paddle is currently using [Git-flow branching model](http://nvie.com/posts/a-successful-git-branching-model/). - -All feature and bug fix development work should be done on a new branch, generally create new branch from `develop` branch . - -```bash -➜ git checkout -b my-cool-stuff -``` - -Before the checkout, you need to keep the current branch directory clean, otherwise the untracked file will be brought to the new branch, which can be inspected by `git status`. - -## Using `pre-commit` hook - -Paddle developers use [pre-commit](http://pre-commit.com/) tool to manage git -pre-commit hooks. It can help us format source codes (cpp, python), check some -basic thing before commit (only one EOL for each file, do not add a huge file -in git). `pre-commit` tests is a part of unit tests in Travis-CI now, every -PR doesn't fit hook can not be merged into Paddle. - -To use [pre-commit](http://pre-commit.com/), you should install it by -`pip install pre-commit`, and currently, Paddle uses `clang-format` to format -c/cpp sources. Please make sure clang-format 3.8+ installed. - -Install and run it as follow: - -```bash -➜ pip install pre-commit -➜ pre-commit install -``` - -When you commit your code, the pre-commit hook will check the local code if there is -anything not suitable to commit, and so on. - -## Start to develop - -In this tutorial, I delete a line in README.md and created a new file. - -We can use `git status` to inspect the changes of current directory, `git diff` to see difference. - -```bash -➜ git status -On branch test -Changes not staged for commit: - (use "git add ..." to update what will be committed) - (use "git checkout -- ..." to discard changes in working directory) - - modified: README.md - -Untracked files: - (use "git add ..." to include in what will be committed) - - test - -no changes added to commit (use "git add" and/or "git commit -a") -``` -## Build and Test - -We package PaddlePaddle's compile environment into a Docker image, called the develop image named `paddle:dev`, it contains all compiling tools that PaddlePaddle needs. - -If you want to build the develop image, just run: - -```bash -➜ docker build -t paddle:dev . -``` - -Then we can use the develop image to build PaddlePaddle source. For example: - -```bash -➜ docker run -v $(pwd):/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=ON" paddle:dev -``` - -The above command will compile PaddlePaddle and create a Dockerfile for building production image. All the generated files are in the build directory. "WITH_GPU" controls if the generated production image supports GPU. "WITH_AVX" controls if the generated production image supports AVX. "WITH_TEST" controls if the unit test will be generated. - -Then we can generate the production image by copying the compiled PaddlePaddle program into the image by - -```bash -➜ docker build -t paddle:prod -f build/Dockerfile . -``` - -Run unit test finally: - -```bash -➜ docker run -it -v $(pwd):/paddle paddle:dev bash -c "cd /paddle/build && ctest" -``` - -For more details, you can read [this doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst). - -## Commit - -Next we cancel the changes to the README.md file and then commit our changes by following command lines: - -```bash -➜ git checkout -- README.md -➜ git status -On branch test -Untracked files: - (use "git add ..." to include in what will be committed) - - test - -nothing added to commit but untracked files present (use "git add" to track) -➜ git add test -``` - -We should write a description of each commit by `git commit` to allow others to know -the changes in these files. - -```bash -➜ git commit -CRLF end-lines remover...............................(no files to check)Skipped -yapf.................................................(no files to check)Skipped -Check for added large files..............................................Passed -Check for merge conflicts................................................Passed -Check for broken symlinks................................................Passed -Detect Private Key...................................(no files to check)Skipped -Fix End of Files.....................................(no files to check)Skipped -clang-formater.......................................(no files to check)Skipped -[my-cool-stuff c703c041] add test file - 1 file changed, 0 insertions(+), 0 deletions(-) - create mode 100644 233 -``` - -## Keeping Fork Up to Date - -Before pull your request, you should sync your code from the latest PaddlePaddle. -To do this, you'll need to add a remote at first: - -```bash -➜ git remote add upstream https://github.com/PaddlePaddle/Paddle -➜ git remote -origin -upstream -``` - -Update your fork with the latest upstream changes: - -```bash -➜ git fetch upstream -➜ git pull upstream develop -``` - -Now, your local master branch is up-to-date with everything modified upstream. - -## Push to GitHub - -```bash -# push to your repository in Github -➜ git push origin my-cool-stuff -``` - -## Create an issue and a Pull Request - -Create an Issue to describe the problem and record its number. - -Go to the page for your fork on GitHub, select your development branch, -and click the `New pull request`. - -screen shot 2017-04-26 at 9 09 28 pm - -Then select the target branch: - -screen shot 2017-04-26 at 9 11 52 pm - -We can add `resolve #Issue number` in PR description to close the issue automatically after the PR is merge. More details in . - -Then wait for review, if there need to modify, refer to the above steps to update the corresponding origin branch. - -## Delete origin branch - -After the PR is merge into the main repository, we can delete the remote branch on the PR page. - -screen shot 2017-04-26 at 9 18 24 pm - -Or just run: - -```bash -➜ git push origin :my-cool-stuff -``` - -## Delete local branch - -Finally, we delete local branch: - -```bash -➜ git checkout develop - -# delete my-cool-stuff branch -➜ git branch -D my-cool-stuff -``` diff --git a/doc/howto/dev/contribute_to_paddle_en.md b/doc/howto/dev/contribute_to_paddle_en.md new file mode 120000 index 0000000000000000000000000000000000000000..c97564d93a7f0a753a23cd97d2467d595bd154ff --- /dev/null +++ b/doc/howto/dev/contribute_to_paddle_en.md @@ -0,0 +1 @@ +../../../CONTRIBUTING.md \ No newline at end of file diff --git a/doc/howto/index_cn.rst b/doc/howto/index_cn.rst index 0608aa30968b0e8474eef330e4d2cc63c9def97d..76d3e0a0092f89005605a23e14e712530112a5ac 100644 --- a/doc/howto/index_cn.rst +++ b/doc/howto/index_cn.rst @@ -21,7 +21,6 @@ dev/build_cn.rst dev/write_docs_cn.rst - dev/contribute_to_paddle_cn.md 模型配置 -------- diff --git a/doc/howto/usage/cluster/cluster_train_cn.md b/doc/howto/usage/cluster/cluster_train_cn.md index 93c5544bcfa911f8bdcdaea39a75b3ab7ef218f8..2e98b3de3fe2284375f87e883ff4bac19255dbeb 100644 --- a/doc/howto/usage/cluster/cluster_train_cn.md +++ b/doc/howto/usage/cluster/cluster_train_cn.md @@ -19,7 +19,7 @@ * [启动集群作业](#启动集群作业-1) * [在Kubernetes集群中提交训练作业](#在kubernetes集群中提交训练作业) -# 概述 +## 概述 本文将介绍如何使用PaddlePaddle在不同的集群框架下完成分布式训练。分布式训练架构如下图所示： @@ -32,7 +32,7 @@ 在使用同步SGD训练神经网络时，PaddlePaddle使用同步屏障（barrier），使梯度的提交和参数的更新按照顺序方式执行。在异步SGD中，则并不会等待所有trainer提交梯度才更新参数，这样极大地提高了计算的并行性：参数服务器之间不相互依赖，并行地接收梯度和更新参数，参数服务器也不会等待计算节点全部都提交梯度之后才开始下一步，计算节点之间也不会相互依赖，并行地执行模型的训练。可以看出，虽然异步SGD方式会提高参数更新并行度, 但是并不能保证参数同步更新，在任意时间某一台参数服务器上保存的参数可能比另一台要更新，与同步SGD相比，梯度会有噪声。 -# 环境准备 +## 环境准备 1. 准备您的计算集群。计算集群通常由一组（几台到几千台规模）的Linux服务器组成。服务器之间可以通过局域网（LAN）联通，每台服务器具有集群中唯一的IP地址（或者可被DNS解析的主机名）。集群中的每台计算机通常被成为一个“节点”。 1. 我们需要在集群的所有节点上安装 PaddlePaddle。 如果要启用GPU，还需要在节点上安装对应的GPU驱动以及CUDA。PaddlePaddle的安装可以参考[build_and_install](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install)的多种安装方式。我们推荐使用[Docker](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_cn.rst)安装方式来快速安装PaddlePaddle。 @@ -51,8 +51,8 @@ PaddlePaddle 0.10.0, compiled with 下面以`doc/howto/usage/cluster/src/word2vec`中的代码作为实例，介绍使用PaddlePaddle v2 API完成分布式训练。 -# 启动参数说明 -## 启动参数服务器 +## 启动参数说明 +### 启动参数服务器 执行以下的命令启动一个参数服务器并等待和计算节点的数据交互 ```bash $ paddle pserver --port=7164 --ports_num=1 --ports_num_for_sparse=1 --num_gradient_servers=1 @@ -70,7 +70,7 @@ $ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num | ports_num_for_sparse | 必选 | 1 | 用于稀疏类型参数通信的端口个数 | | num_gradient_servers | 必选 | 1 | 当前训练任务pserver总数 | -## 启动计算节点 +### 启动计算节点 执行以下命令启动使用python编写的trainer程序（文件名为任意文件名，如train.py） ```bash $ python train.py @@ -117,7 +117,7 @@ paddle.init( | pservers | 必选 | 127.0.0.1 | 当前训练任务启动的pserver的IP列表，多个IP使用“,”隔开 | -## 准备数据集 +### 准备数据集 参考样例数据准备脚本[prepare.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py)，准备训练数据和验证数据集，我们使用paddle.dataset.imikolov数据集，并根据分布式训练并发数（trainer节点个数），在`prepare.py`开头部分指定`SPLIT_COUNT`将数据切分成多份。 @@ -149,7 +149,7 @@ test.txt-00002 对于不同的训练任务，训练数据格式和训练程序的`reader()`会大不相同，所以开发者需要根据自己训练任务的实际场景完成训练数据的分割和`reader()`的编写。 -## 准备训练程序 +### 准备训练程序 我们会对每个训练任务都会在每个节点上创建一个工作空间（workspace），其中包含了用户的训练程序、程序依赖、挂载或下载的训练数据分片。 @@ -184,7 +184,7 @@ test.txt-00002 - `train_data_dir`：包含训练数据的目录，可以是从分布式存储挂载过来的，也可以是在任务启动前下载到本地的。 - `test_data_dir`：包含测试数据集的目录。 -# 使用分布式计算平台或工具 +## 使用分布式计算平台或工具 PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务，包括： - [Kubernetes](http://kubernetes.io) Google开源的容器集群的调度框架，支持大规模集群生产环境的完整集群方案。 @@ -195,12 +195,12 @@ PaddlePaddle可以使用多种分布式计算平台构建分布式计算任务 在使用分布式计算平台进行训练时，任务被调度在集群中时，分布式计算平台通常会通过API或者环境变量提供任务运行需要的参数，比如节点的ID、IP和任务节点个数等。 -## 使用Fabric启动集群作业 +### 使用Fabric启动集群作业 -### 准备一个Linux集群 +#### 准备一个Linux集群 可以在`paddle/scripts/cluster_train_v2/fabric/docker_cluster`目录下，执行`kubectl -f ssh_servers.yaml`启动一个测试集群，并使用`kubectl get po -o wide`获得这些节点的IP地址。 -### 启动集群作业 +#### 启动集群作业 `paddle.py` 提供了自动化脚本来启动不同节点中的所有 PaddlePaddle 集群进程。默认情况下，所有命令行选项可以设置为 `paddle.py` 命令选项并且 `paddle.py` 将透明、自动地将这些选项应用到 PaddlePaddle 底层进程。 @@ -216,10 +216,10 @@ sh run.sh 集群作业将会在几秒后启动。 -### 终止集群作业 +#### 终止集群作业 `paddle.py`能获取`Ctrl + C` SIGINT 信号来自动终止它启动的所有进程。只需中断 `paddle.py` 任务来终止集群作业。如果程序崩溃你也可以手动终止。 -### 检查集群训练结果 +#### 检查集群训练结果 详细信息请检查 $workspace/log 里的日志，每一个节点都有相同的日志结构。 `paddle_trainer.INFO` @@ -234,13 +234,13 @@ sh run.sh `train.log` 提供训练过程的 stderr 和 stdout。训练失败时可以检查错误日志。 -### 检查模型输出 +#### 检查模型输出 运行完成后，模型文件将被写入节点 0 的 `output` 目录中。 工作空间中的 `nodefile` 表示当前集群作业的节点 ID。 -## 在OpenMPI集群中提交训练作业 +### 在OpenMPI集群中提交训练作业 -### 准备OpenMPI集群 +#### 准备OpenMPI集群 执行下面的命令以启动3个节点的OpenMPI集群和一个"head"节点： @@ -252,7 +252,7 @@ kubectl create -f mpi-nodes.yaml 然后可以从head节点ssh无密码登录到OpenMPI的每个节点上。 -### 启动集群作业 +#### 启动集群作业 您可以按照下面的步骤在OpenMPI集群中提交paddle训练任务： @@ -280,6 +280,6 @@ scp train.txt-00002 test.txt-00002 [node3IP]:/home/tutorial mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ``` -## 在Kubernetes集群中提交训练作业 +### 在Kubernetes集群中提交训练作业 此部分的使用方法可以参考[here](../k8s/k8s_distributed_cn.md)。 diff --git a/doc/howto/usage/cluster/cluster_train_en.md b/doc/howto/usage/cluster/cluster_train_en.md index 1e8b4d54b9ffa99b3beef35ecaf95bbd0866535f..baa97c0c02ae490fff8587071bd2d4adfb5325e3 100644 --- a/doc/howto/usage/cluster/cluster_train_en.md +++ b/doc/howto/usage/cluster/cluster_train_en.md @@ -19,7 +19,7 @@ * [Launching Cluster Job](#launching-cluster-job-1) * [Cluster Training Using Kubernetes](#cluster-training-using-kubernetes) -# Introduction +## Introduction In this article, we'll explain how to run distributed training jobs with PaddlePaddle on different types of clusters. The diagram below shows the main architecture of a distributed trainning job: @@ -33,7 +33,7 @@ PaddlePaddle can support both synchronize stochastic gradient descent (SGD) and When training with synchronize SGD, PaddlePaddle uses an internal "synchronize barrier" which makes gradients update and parameter download in strict order. On the other hand, asynchronous SGD won't wait for all trainers to finish upload at a single step, this will increase the parallelism of distributed training: parameter servers do not depend on each other, they'll do parameter optimization concurrently. Parameter servers will not wait for trainers, so trainers will also do their work concurrently. But asynchronous SGD will introduce more randomness and noises in the gradient. -# Preparations +## Preparations 1. Prepare your computer cluster. It's normally a bunch of Linux servers connected by LAN. Each server will be assigned a unique IP address. The computers in the cluster can be called "nodes". 2. Install PaddlePaddle on every node. If you are going to take advantage of GPU cards, you'll also need to install proper driver and CUDA libraries. To install PaddlePaddle please read [this build and install](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/getstarted/build_and_install) document. We strongly recommend using [Docker installation](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/getstarted/build_and_install/docker_install_en.rst). @@ -52,9 +52,9 @@ PaddlePaddle 0.10.0rc, compiled with We'll take `doc/howto/usage/cluster/src/word2vec` as an example to introduce distributed training using PaddlePaddle v2 API. -# Command-line arguments +## Command-line arguments -## Starting parameter server +### Starting parameter server Type the below command to start a parameter server which will wait for trainers to connect: @@ -74,7 +74,7 @@ $ stdbuf -oL /usr/bin/nohup paddle pserver --port=7164 --ports_num=1 --ports_num | ports_num_for_sparse | required | 1 | number of ports which serves sparse parameter update | | num_gradient_servers | required | 1 | total number of gradient servers | -## Starting trainer +### Starting trainer Type the command below to start the trainer(name the file whatever you want, like "train.py") ```bash @@ -122,7 +122,7 @@ paddle.init( | trainer_id | required | 0 | ID for every trainer, start from 0 | | pservers | required | 127.0.0.1 | list of IPs of parameter servers, separated by "," | -## Prepare Training Dataset +### Prepare Training Dataset Here's some example code [prepare.py](https://github.com/PaddlePaddle/Paddle/tree/develop/doc/howto/usage/cluster/src/word2vec/prepare.py), it will download public `imikolov` dataset and split it into multiple files according to job parallelism(trainers count). Modify `SPLIT_COUNT` at the begining of `prepare.py` to change the count of output files. @@ -155,7 +155,7 @@ When job started, every trainer needs to get it's own part of data. In some dist Different training jobs may have different data format and `reader()` function, developers may need to write different data prepare scripts and `reader()` functions for their job. -## Prepare Training program +### Prepare Training program We'll create a *workspace* directory on each node, storing your training program, dependencies, mounted or downloaded dataset directory. @@ -191,7 +191,7 @@ Your workspace may looks like: - `train_data_dir`: containing training data. Mount from storage service or copy trainning data to here. - `test_data_dir`: containing testing data. -# Use cluster platforms or cluster management tools +## Use cluster platforms or cluster management tools PaddlePaddle supports running jobs on several platforms including: - [Kubernetes](http://kubernetes.io) open-source system for automating deployment, scaling, and management of containerized applications from Google. @@ -202,13 +202,13 @@ We'll introduce cluster job management on these platforms. The examples can be f These cluster platforms provide API or environment variables for training processes, when the job is dispatched to different nodes. Like node ID, IP or total number of nodes etc. -## Cluster Training Using Fabric +### Cluster Training Using Fabric -### Prepare a Linux cluster +#### Prepare a Linux cluster Run `kubectl -f ssh_servers.yaml` under the directory: `paddle/scripts/cluster_train_v2/fabric/docker_cluster` will launch a demo cluster. Run `kubectl get po -o wide` to get IP addresses of these nodes. -### Launching Cluster Job +#### Launching Cluster Job `paddle.py` provides automatical scripts to start all PaddlePaddle cluster processes in different nodes. By default, all command line options can be set as `paddle.py` command options and `paddle.py` will transparently and automatically set these options to PaddlePaddle lower level processes. `paddle.py`provides two distinguished command option for easy job launching. @@ -224,10 +224,10 @@ sh run.sh The cluster Job will start in several seconds. -### Kill Cluster Job +#### Kill Cluster Job `paddle.py` can capture `Ctrl + C` SIGINT signal to automatically kill all processes launched by it. So just stop `paddle.py` to kill cluster job. You should manually kill the job if the program crashed. -### Check Cluster Training Result +#### Check Cluster Training Result Check log in $workspace/log for details, each node owns same log structure. `paddle_trainer.INFO` @@ -242,13 +242,13 @@ It provides stderr and stdout of parameter server process. Check error log if tr `train.log` It provides stderr and stdout of trainer process. Check error log if training crashes. -### Check Model Output +#### Check Model Output After one pass finished, model files will be written in `output` directory in node 0. `nodefile` in workspace indicates the node id of current cluster job. -## Cluster Training Using OpenMPI +### Cluster Training Using OpenMPI -### Prepare an OpenMPI cluster +#### Prepare an OpenMPI cluster Run the following command to start a 3-node MPI cluster and one "head" node. @@ -260,7 +260,7 @@ kubectl create -f mpi-nodes.yaml Then you can log in to every OpenMPI node using ssh without input any passwords. -### Launching Cluster Job +#### Launching Cluster Job Follow the steps to launch a PaddlePaddle training job in OpenMPI cluster:\ @@ -288,6 +288,6 @@ scp train.txt-00002 test.txt-00002 [node3IP]:/home/tutorial mpirun -hostfile machines -n 3 /home/tutorial/start_mpi_train.sh ``` -## Cluster Training Using Kubernetes +### Cluster Training Using Kubernetes The details can be found [here](../k8s/k8s_cn.md) diff --git a/go/.gitignore b/go/.gitignore index 000e1fd55b63b8e532308b787c2708a6c3e5ac87..398d70ca375ffceccdbfc82a4851a6830ca31264 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -1,2 +1,3 @@ vendor/ .glide/ +proto/*.go diff --git a/go/glide.lock b/go/glide.lock index ce654d36364f8078a493651d8d8b141532eea26d..d15fc934dbe511389cc92ce95cededa41ba32b4d 100644 --- a/go/glide.lock +++ b/go/glide.lock @@ -1,5 +1,5 @@ -hash: 51d9e2e46d7fd9173ff11ecada40f7b7728756be18d5e2f032535f66465e6e15 -updated: 2017-10-24T15:04:09.987751592-07:00 +hash: 107c058cf5c9163a75d40eef2273a793c36112683c25d72aa8288827fdde3a19 +updated: 2017-10-30T03:46:19.137696069Z imports: - name: github.com/alecthomas/gometalinter version: bae2f1293d092fd8167939d5108d1b025eaef9de diff --git a/go/glide.yaml b/go/glide.yaml index ba253f8bebef0ddab810a8303ab1fbe541defbdf..c5d66694acd0f45de5002391a7953b7491eaf2bc 100644 --- a/go/glide.yaml +++ b/go/glide.yaml @@ -30,3 +30,4 @@ import: version: v2.13 - package: github.com/go-stack/stack version: v1.6.0 +- package: github.com/golang/protobuf diff --git a/go/proto/.gitignore b/go/proto/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..5e7d2734cfc60289debf74293817c0a8f572ff32 --- /dev/null +++ b/go/proto/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/go/pserver/CMakeLists.txt b/go/pserver/CMakeLists.txt index 4fe0a8cb021e8dbf443c8f33bfb046e228a2fd8d..9ac05199e7ab76c21275838092c0afbdf2612b77 100644 --- a/go/pserver/CMakeLists.txt +++ b/go/pserver/CMakeLists.txt @@ -13,5 +13,5 @@ # limitations under the License. # if(WITH_TESTING) - go_test(pserver_test DEPS paddle_go_optimizer) + go_test(pserver_test DEPS paddle_go_optimizer gen_proto_go) endif() diff --git a/go/pserver/service.go b/go/pserver/service.go index f703d99a29ae9f5310ef36a7492b729c4c892937..7484ec90b1a3a9e67fa798741a9dfeb580c51f1a 100644 --- a/go/pserver/service.go +++ b/go/pserver/service.go @@ -17,6 +17,7 @@ package pserver import ( "bufio" "bytes" + "encoding/binary" "encoding/gob" "encoding/json" "errors" @@ -26,11 +27,15 @@ import ( "os" "path" "strconv" + "strings" "sync" "time" + "github.com/golang/protobuf/proto" uuid "github.com/satori/go.uuid" + pb "github.com/PaddlePaddle/Paddle/go/proto" + log "github.com/inconshreveable/log15" ) @@ -65,6 +70,46 @@ type Parameter struct { Content []byte } +func float32ToString(b []byte) string { + f := make([]float32, len(b)/4) + buf := bytes.NewReader(b) + err := binary.Read(buf, binary.LittleEndian, &f) + if err != nil { + return "" + } + return fmt.Sprintf("%v", f) +} + +func float32ByteToString(c []byte) string { + var a []byte + var b []byte + if len(c) <= 80 { + a = c + } else { + a = c[0:40] + b = c[len(c)-40:] + } + + var s string + s = float32ToString(a) + + if b == nil { + return s + } + + s = strings.Replace(s, "]", "", -1) + "..." + strings.Replace(float32ToString(b), "[", "", -1) + return s +} + +func (p Parameter) String() string { + if p.ElementType != Float32 { + return fmt.Sprintf("name:%v ElementType:%v", + p.Name, p.ElementType) + } + + return float32ByteToString(p.Content) +} + // ParameterWithConfig contains the parameter and the configuration. type ParameterWithConfig struct { Param Parameter @@ -189,7 +234,9 @@ func (s *Service) InitParam(paramWithConfigs ParameterWithConfig, _ *int) error default: } - // TODO(helin): parse parameter config + c := &pb.OptimizerConfig{} + proto.Unmarshal(paramWithConfigs.Config, c) + log.Debug(fmt.Sprintf("OptimizerConfig:%v", c)) s.mu.Lock() defer s.mu.Unlock() @@ -239,7 +286,8 @@ func (s *Service) SendGrad(g Gradient, _ *int) error { select { case <-s.initialized: default: - log.Warn("received gradient before initialization.", "name", g.Name, "size", len(g.Content), "type", g.ElementType) + log.Warn("received gradient before initialization.", + "name", g.Name, "size", len(g.Content), "type", g.ElementType) return errors.New(Uninitialized) } @@ -248,10 +296,14 @@ func (s *Service) SendGrad(g Gradient, _ *int) error { o, ok := s.optMap[g.Name] if !ok { + log.Warn("received gradient but can't find name.", + "name", g.Name, "size", len(g.Content), "type", g.ElementType) return fmt.Errorf("parameter: %s does not exist", g.Name) } - log.Info("received gradient from trainer, updating gradient.", "name", g.Name, "size", len(g.Content), "type", g.ElementType) + log.Debug(Parameter(g).String()) + log.Info("received gradient from trainer, updating gradient.", + "name", g.Name, "size", len(g.Content), "type", g.ElementType) return o.UpdateParameter(g) } @@ -277,7 +329,7 @@ func (s *Service) GetParam(name string, parameter *Parameter) error { parameter.Name = name parameter.ElementType = opt.elementType parameter.Content = opt.GetWeights() - + log.Debug(parameter.String()) log.Info("sending parameter to the trainer", "name", parameter.Name, "size", len(parameter.Content), "type", parameter.ElementType) return nil } diff --git a/go/pserver/service_test.go b/go/pserver/service_test.go index b6f4566eb78cf797e3738afa5f86f5c4e8090d85..58a743e1fadff9d629f682d660e661013c33ac8a 100644 --- a/go/pserver/service_test.go +++ b/go/pserver/service_test.go @@ -15,6 +15,7 @@ package pserver_test import ( + "fmt" "io/ioutil" "reflect" "sync" @@ -178,3 +179,33 @@ func TestBlockUntilInitialized(t *testing.T) { wg.Wait() } + +func TestGradientString(t *testing.T) { + g := pserver.Parameter{} + g.ElementType = pserver.Float32 + g.Content = []byte{0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40} + if g.String() != "[3.3702806e+12 2.142699 3.3702806e+12 2.142699]" { + t.Fatal("get float data error!") + } + + g.Content = []byte{0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40, + 0x18, 0x2d, 0x44, 0x54, 0xfb, 0x21, 0x09, 0x40} + if g.String() != "[3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699...3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699 3.3702806e+12 2.142699]" { + t.Fatal("get float data error!", g.String()) + } + fmt.Println(g) +} diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index c7f25109972195fb56b9e96c4b68d952363e6338..7daca18761b80eac0f876b21377a6ccc6a853485 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -300,4 +300,12 @@ extern void hl_matrix_col2Vol(real* dataDst, real alpha, real beta); +/** + * @brief Matrix col2Vol: Convert col matrix into 3D volume + * @param[out] out output int vector. + * @param[in] vec input float vector. + * @param[in] size size of the vector. + */ +extern void hl_vector_cast2int(int* out, real* vec, int size); + #endif /* HL_MATRIX_H_ */ diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index 6ac332945c8f09fef23f35680ba5bb1d9ba9f4fd..46e77e140768dd80fd327dd4eb3b0f62a3370950 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -133,4 +133,6 @@ inline void hl_matrix_col2Vol(real* dataDst, real alpha, real beta) {} +inline void hl_vector_cast2int(int* out, real* vec, int size) {} + #endif // HL_MATRIX_STUB_H_ diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index b41a3a1e06db7b2566acef19ce430645f79d486d..607efb4f6b0aa0d22a2789397b8743f7a5271d5b 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -793,3 +793,14 @@ void hl_matrix_col2Vol(real* dataDst, CHECK_SYNC("hl_matrix_col2Vol failed"); } + +__global__ void keVectorCast2Int(int* out, real* vec, int size) { + for (int i = threadIdx.x; i < (size); i += blockDim.x) { + out[i] = int(vec[i]); + } +} + +void hl_vector_cast2int(int* out, real* vec, int size) { + keVectorCast2Int<<<1, 512, 0, STREAM_DEFAULT>>>(out, vec, size); + CHECK_SYNC("hl_vector_cast2int failed"); +} diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index f4fef055daf39e9be0645deaafdad4132fc7e35f..1afc5242081e7f7b12527a15d29421cebeb3d3b8 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -20,7 +20,8 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) cc_library(attribute SRCS attribute.cc DEPS framework_proto) -cc_test(program_desc_test SRCS program_desc_test.cc DEPS proto_desc) +cc_test(program_desc_test SRCS program_desc_test.cc DEPS proto_desc +device_context) cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) @@ -44,8 +45,9 @@ add_custom_command(TARGET framework_py_proto POST_BUILD cc_library(backward SRCS backward.cc DEPS net_op) cc_test(backward_test SRCS backward_test.cc DEPS backward recurrent_op device_context fill_constant_op) +cc_library(lod_rank_table SRCS lod_rank_table.cc DEPS lod_tensor) -cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog) +cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog lod_rank_table) cc_library(prune SRCS prune.cc DEPS framework_proto) cc_test(prune_test SRCS prune_test.cc DEPS op_info prune recurrent_op device_context) diff --git a/paddle/framework/attribute.cc b/paddle/framework/attribute.cc index 29fe352ca450740e55ee87b63392e3aabac8aa40..b1e17936417e4ce09bace1d1a5d346d1c9cfa710 100644 --- a/paddle/framework/attribute.cc +++ b/paddle/framework/attribute.cc @@ -19,7 +19,7 @@ limitations under the License. */ namespace paddle { namespace framework { -Attribute GetAttrValue(const OpDesc::Attr& attr_desc, ProgramDesc* program) { +Attribute GetAttrValue(const OpDesc::Attr& attr_desc) { switch (attr_desc.type()) { case framework::AttrType::BOOLEAN: { return attr_desc.b(); @@ -61,13 +61,9 @@ Attribute GetAttrValue(const OpDesc::Attr& attr_desc, ProgramDesc* program) { } return val; } - case framework::AttrType::BLOCK: { - PADDLE_ENFORCE(program != nullptr, - "Need to specify ProgramDesc when get a block attr"); - return program->mutable_blocks(attr_desc.block_idx()); - } + default: + PADDLE_THROW("Unsupport attr type %d", attr_desc.type()); } - PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !"); return boost::blank(); } diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index 9744662b8f7229b0b17e910ae5cd997fa7d31e06..0641907d6ff7546df1601d3b0263ff42f4186968 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -32,7 +32,7 @@ inline AttrType AttrTypeID() { return static_cast(tmp.which() - 1); } -Attribute GetAttrValue(const OpDesc::Attr& attr_desc, ProgramDesc* desc); +Attribute GetAttrValue(const OpDesc::Attr& attr_desc); class AttrReader { public: diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 150c152367e1bcdc095bce6f77fafdef601e1c47..ed94540c268e5ed990c1d92859c6a2093c052868 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -18,12 +18,12 @@ #include #include #include +#include #include "paddle/framework/block_desc.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/dynamic_recurrent_op.h" #include "paddle/operators/net_op.h" -#include "paddle/operators/recurrent_op.h" namespace paddle { namespace framework { @@ -37,7 +37,7 @@ static inline std::unique_ptr CreateGradOp( op_desc.SetType(op.Type()); op_desc.SetAttrMap(op.Attrs()); auto& info = OpInfoMap::Instance().Get(op.Type()); - auto grad_descs = info.GradOpMaker()(op_desc, no_grad_set, grad_to_var); + auto grad_descs = info.GradOpMaker()(op_desc, no_grad_set, grad_to_var, {}); std::vector> grad_ops; grad_ops.reserve(grad_descs.size()); std::transform(grad_descs.begin(), grad_descs.end(), @@ -219,19 +219,7 @@ static std::unique_ptr BackwardRecursive( }); // process recurrent gradient op as a special operator. - if (forwardOp.Type() == "recurrent") { - // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), - // or this will result in infinite loop. - const auto& rnnop = - *static_cast(&forwardOp); - auto rnn_grad_op = - static_cast(grad_op.get()); - const auto& stepnet_op = - *static_cast(&rnnop.stepnet()); - // create stepnet's gradient op - rnn_grad_op->set_stepnet( - BackwardRecursive(stepnet_op, no_grad_names, grad_to_var, uniq_id)); - } else if (forwardOp.Type() == "dynamic_recurrent") { + if (forwardOp.Type() == "dynamic_recurrent") { // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), // or this will result in infinite loop. const auto& rnnop = @@ -285,6 +273,15 @@ static bool AllGradInSet(const std::vector& names, return true; } +static std::string FwdName(const std::string& grad_name) { + auto pos = grad_name.find("@GRAD"); + if (pos == std::string::npos) { + return ""; + } else { + return grad_name.substr(0, pos); + } +} + static void CreateGradVarInBlock( size_t grad_op_start_index, const std::unordered_map& param_name_map, @@ -294,6 +291,7 @@ static void CreateGradVarInBlock( for (size_t op_index = grad_op_start_index; op_index < ops.size(); ++op_index) { bool need_infer_shape = false; + std::unordered_set new_vars; ForEachVarName(ops[op_index]->Outputs(), [&](const std::string& grad_var_name) { if (block_desc->HasVar(grad_var_name)) { @@ -301,8 +299,7 @@ static void CreateGradVarInBlock( } need_infer_shape = true; auto var = block_desc->Var(grad_var_name); - // FIXME(qiao) infer the datatype - var->SetDataType(framework::DataType::FP32); + new_vars.insert(var->Name()); auto it = param_name_map.find(grad_var_name); if (it == param_name_map.end()) { return false; @@ -316,6 +313,21 @@ static void CreateGradVarInBlock( }); if (need_infer_shape) { ops[op_index]->InferVarType(block_desc); + for (auto& arg : ops[op_index]->OutputArgumentNames()) { + if (new_vars.find(arg) == new_vars.end()) { + continue; + } + auto pname = FwdName(arg); + auto* param = block_desc->FindVarRecursive(pname); + auto* grad = block_desc->FindVar(arg); + if (param == nullptr) { + LOG(WARNING) << "Cannot find forward variable of " << arg + << ". Set its gradient to FP32"; + grad->SetDataType(DataType::FP32); + } else { + grad->SetDataType(param->GetDataType()); + } + } ops[op_index]->InferShape(*block_desc); } } @@ -323,7 +335,9 @@ static void CreateGradVarInBlock( std::vector> MakeOpGrad( const OpDescBind* op_desc, std::unordered_set* no_grad_vars, - std::unordered_map* grad_to_var) { + std::unordered_map* grad_to_var, + const std::vector& grad_block = + std::vector()) { std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculate. const std::vector& inputs = op_desc->InputArgumentNames(); @@ -339,9 +353,10 @@ std::vector> MakeOpGrad( return grad_op_descs; // empty vector } - grad_op_descs = OpInfoMap::Instance() - .Get(op_desc->Type()) - .GradOpMaker()(*op_desc, *no_grad_vars, grad_to_var); + grad_op_descs = + OpInfoMap::Instance() + .Get(op_desc->Type()) + .GradOpMaker()(*op_desc, *no_grad_vars, grad_to_var, grad_block); std::list> pending_fill_zeros_ops; for (auto& desc : grad_op_descs) { @@ -368,28 +383,27 @@ std::vector> MakeBlockBackward( ProgramDescBind& program_desc, int block_idx, std::unordered_set* no_grad_vars, std::unordered_map* grad_to_var) { - BlockDescBind* cur_block = program_desc.Block(block_idx); + BlockDescBind* cur_block = program_desc.MutableBlock(block_idx); std::vector op_descs = cur_block->AllOps(); std::unordered_map> dup_out_ops; size_t grad_desc_idx = 0; std::vector> backward_descs; for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { - std::vector> op_grads = - MakeOpGrad(*it, no_grad_vars, grad_to_var); + std::vector> op_grads; if ((*it)->Type() == "recurrent") { - PADDLE_ENFORCE_EQ( - op_grads.size(), static_cast(1), - "rnn_op's gradient process should contain only one op."); int step_block_idx = (*it)->GetBlockAttr("step_block"); auto backward_block_op_descs = MakeBlockBackward( program_desc, step_block_idx, no_grad_vars, grad_to_var); - BlockDescBind* backward_block = program_desc.AppendBlock(*cur_block); + BlockDescBind* backward_block = + program_desc.AppendBlock(*program_desc.MutableBlock(step_block_idx)); for (auto& ptr : backward_block_op_descs) { backward_block->AppendAllocatedOp(std::move(ptr)); } - op_grads[0]->SetBlockAttr("step_block", *backward_block); + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); + } else { + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var); } for (const auto& desc : op_grads) { @@ -443,7 +457,7 @@ ParamGradInfoMap AppendBackward( } const int root_block_idx = 0; - auto root_block = program_desc.Block(root_block_idx); + auto root_block = program_desc.MutableBlock(root_block_idx); // insert fill one op for target // TODO(qiao) add some check to the target. @@ -492,7 +506,7 @@ ParamGradInfoMap AppendBackward( CreateGradVarInBlock(forward_op_num, grad_to_var, root_block, &retv); for (size_t block_index = forward_block_num; block_index < program_desc.Size(); ++block_index) { - CreateGradVarInBlock(0, grad_to_var, program_desc.Block(block_index), + CreateGradVarInBlock(0, grad_to_var, program_desc.MutableBlock(block_index), &retv); } return retv; diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 421f1321948235aa0c1acd2e24037b34716e449a..4e8d630c2634682ff63b38182108eadebb5c7ff9 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -499,7 +499,7 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) { TEST(Backward, simple_single_op) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op = block->AppendOp(); op->SetType("rowwise_add"); @@ -535,7 +535,7 @@ TEST(Backward, simple_single_op) { TEST(Backward, default_attribute) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op = block->AppendOp(); op->SetType("mul"); op->SetInput("X", {"x"}); @@ -561,7 +561,7 @@ TEST(Backward, default_attribute) { TEST(Backward, simple_mult_op) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); @@ -644,7 +644,7 @@ TEST(Backward, simple_mult_op) { TEST(Backward, intermedia_var_no_grad) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); @@ -714,7 +714,7 @@ TEST(Backward, intermedia_var_no_grad) { TEST(Backward, var_no_grad) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op1 = block->AppendOp(); op1->SetType("mult_in_out"); op1->SetInput("X", {"x1"}); @@ -790,7 +790,7 @@ TEST(Backward, var_no_grad) { TEST(Backward, shared_var) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); f::OpDescBind *op1 = block->AppendOp(); op1->SetType("rowwise_add"); op1->SetInput("X", {"x1"}); @@ -880,7 +880,7 @@ TEST(Backward, shared_var) { TEST(Backward, half_backward) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); auto *op1 = block->AppendOp(); op1->SetType("minus"); op1->SetInput("X", {"a"}); diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index b73a20cc89d936c2beee6a39cdf71cda3915bcdc..9e3d597f3a2c84623a1ce9e4b6f4b956cffde211 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -113,7 +113,7 @@ BlockDescBind *BlockDescBind::ParentBlock() const { if (this->desc_->parent_idx() == kNoneBlockIndex) { return nullptr; } - return prog_->Block(static_cast(this->desc_->parent_idx())); + return prog_->MutableBlock(static_cast(this->desc_->parent_idx())); } BlockDesc *BlockDescBind::Proto() { diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 72f77a88a24434fd7d2ed685ac850c88888d6808..26adf6a20ff09483b84f479db08efcf402135053 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -88,6 +88,8 @@ class BlockDescBind { BlockDesc *Proto(); + ProgramDescBind *Program() { return this->prog_; } + private: void ClearPBOps(); void ClearPBVars(); diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index b731840ef2a4b2d5d82b019d28ad6517fa4b7607..f91e0e03410c95f84a65f02beed38b7bbfdcaa86 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -108,8 +108,9 @@ struct OpInfoFiller { info->grad_op_maker_ = []( const OpDescBind& fwd_op, const std::unordered_set& no_grad_set, - std::unordered_map* grad_to_var) { - T maker(fwd_op, no_grad_set, grad_to_var); + std::unordered_map* grad_to_var, + const std::vector& grad_block) { + T maker(fwd_op, no_grad_set, grad_to_var, grad_block); return maker(); }; } diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index 3e9d8b3084e8a76f3d5b8367b0ec45ed74dec42f..2fcf41d69f0011b0d9a3d89c97fcebacb0703e97 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -21,7 +21,9 @@ limitations under the License. */ #include #include "paddle/framework/feed_fetch_type.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/op_registry.h" #include "paddle/framework/scope.h" @@ -31,7 +33,7 @@ namespace framework { const std::string kFeedOpType = "feed"; const std::string kFetchOpType = "fetch"; -Executor::Executor(const std::vector& places) { +Executor::Executor(const std::vector& places) : own_(true) { PADDLE_ENFORCE_GT(places.size(), 0); device_contexts_.resize(places.size()); for (size_t i = 0; i < places.size(); i++) { @@ -52,8 +54,10 @@ Executor::Executor(const std::vector& places) { } Executor::~Executor() { - for (auto& device_context : device_contexts_) { - delete device_context; + if (own_) { + for (auto& device_context : device_contexts_) { + delete device_context; + } } } @@ -66,45 +70,65 @@ static void CreateTensor(Variable* var, VarDesc::VarType var_type) { var->GetMutable(); } else if (var_type == VarDesc::FETCH_LIST) { var->GetMutable(); + } else if (var_type == VarDesc::STEP_SCOPES) { + var->GetMutable>(); + } else if (var_type == VarDesc::LOD_RANK_TABLE) { + var->GetMutable(); + } else if (var_type == VarDesc::LOD_TENSOR_ARRAY) { + var->GetMutable(); } else { PADDLE_THROW( - "Variable type must be " - "LoDTensor/SelectedRows/FEED_MINIBATCH/FETCH_LIST."); + "Variable type %d is not in " + "[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST, LOD_RANK_TABLE]", + var_type); } } -void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id) { +void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, + bool create_local_scope) { // TODO(tonyyang-svail): // - only runs on the first device (i.e. no interdevice communication) // - will change to use multiple blocks for RNN op and Cond Op - PADDLE_ENFORCE_GT(pdesc.blocks_size(), block_id); - auto& block = pdesc.blocks(block_id); + PADDLE_ENFORCE_LT(static_cast(block_id), pdesc.Size()); + auto& block = pdesc.Block(block_id); auto& device = device_contexts_[0]; - Scope& local_scope = scope->NewScope(); - - for (auto& var : block.vars()) { - if (var.persistable()) { - auto* ptr = scope->Var(var.name()); - CreateTensor(ptr, var.type()); - VLOG(3) << "Create Variable " << var.name() - << " global, which pointer is " << ptr; - } else { - auto* ptr = local_scope.Var(var.name()); - CreateTensor(ptr, var.type()); - VLOG(3) << "Create Variable " << var.name() - << " locally, which pointer is " << ptr; + Scope* local_scope = scope; + if (create_local_scope) { + local_scope = &scope->NewScope(); + for (auto& var : block.AllVars()) { + if (var->Persistable()) { + auto* ptr = scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " global, which pointer is " << ptr; + } else { + auto* ptr = local_scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " locally, which pointer is " << ptr; + } + } + } else { + for (auto& var : block.AllVars()) { + auto* ptr = local_scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create variable " << var->Name() << ", which pointer is " + << ptr; } } - for (auto& op_desc : block.ops()) { - auto op = paddle::framework::OpRegistry::CreateOp( - op_desc, const_cast(&pdesc)); - op->Run(local_scope, *device); + for (auto& op_desc : block.AllOps()) { + auto op = paddle::framework::OpRegistry::CreateOp(*op_desc); + op->Run(*local_scope, *device); + } + if (create_local_scope) { + scope->DeleteScope(local_scope); } - - scope->DeleteScope(&local_scope); } +Executor::Executor(const platform::DeviceContext& device) + : device_contexts_({&device}), own_(false) {} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index 793ee954e25f7da6c9d04ea6acc2ad78812e8329..b745f4f6474ef688774f4c833a3958942e9aa8cb 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -14,8 +14,8 @@ limitations under the License. */ #pragma once -#include "paddle/framework/framework.pb.h" #include "paddle/framework/op_info.h" +#include "paddle/framework/program_desc.h" #include "paddle/framework/scope.h" #include "paddle/framework/tensor.h" @@ -25,6 +25,7 @@ namespace framework { class Executor { public: explicit Executor(const std::vector& places); + explicit Executor(const platform::DeviceContext& devices); ~Executor(); /* @Brief @@ -34,10 +35,11 @@ class Executor { * ProgramDesc * Scope */ - void Run(const ProgramDesc&, Scope*, int); + void Run(const ProgramDescBind&, Scope*, int, bool create_local_scope = true); private: - std::vector device_contexts_; + std::vector device_contexts_; + bool own_; }; } // namespace framework diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 8f2df3dc0e29f96b3aea58b6761d1ccb4cd7c624..f1fc4529e15502927560eefd74110f6ca7eab4a9 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -109,6 +109,11 @@ message LoDTensorDesc { optional int32 lod_level = 2 [ default = 0 ]; } +message LoDTensorArrayDesc { + required TensorDesc tensor = 1; + optional int32 lod_level = 2 [ default = 0 ]; +} + message VarDesc { enum VarType { LOD_TENSOR = 1; @@ -116,11 +121,14 @@ message VarDesc { FEED_MINIBATCH = 3; FETCH_LIST = 4; STEP_SCOPES = 5; + LOD_RANK_TABLE = 6; + LOD_TENSOR_ARRAY = 7; } required string name = 1; required VarType type = 2; optional LoDTensorDesc lod_tensor = 3; optional TensorDesc selected_rows = 4; + optional LoDTensorArrayDesc tensor_array = 6; optional bool persistable = 5 [ default = false ]; } diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index 94944c79b64d38e799df436de874cabc3661e30a..998186e33915a11f2864eb5387d19ed1bfbab51c 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -15,6 +15,7 @@ #pragma once #include #include +#include #include "paddle/framework/op_desc.h" #include "paddle/framework/operator.h" @@ -26,8 +27,13 @@ class GradOpDescMakerBase { explicit GradOpDescMakerBase( const OpDescBind& fwd_op, const std::unordered_set& no_grad_set, - std::unordered_map* grad_to_var) - : fwd_op_(fwd_op), no_grad_set_(no_grad_set), grad_to_var_(grad_to_var) {} + std::unordered_map* grad_to_var, + const std::vector& grad_block = + std::vector()) + : fwd_op_(fwd_op), + no_grad_set_(no_grad_set), + grad_to_var_(grad_to_var), + grad_block_(grad_block) {} virtual ~GradOpDescMakerBase() = default; virtual std::vector> operator()() const = 0; @@ -102,6 +108,9 @@ class GradOpDescMakerBase { const OpDescBind& fwd_op_; const std::unordered_set& no_grad_set_; std::unordered_map* grad_to_var_; + + protected: + std::vector grad_block_; }; class SingleGradOpDescMaker : public GradOpDescMakerBase { diff --git a/paddle/framework/lod_rank_table.cc b/paddle/framework/lod_rank_table.cc new file mode 100644 index 0000000000000000000000000000000000000000..68a83def7e5a12fe3261be9e27cbb9bb54e1f8ad --- /dev/null +++ b/paddle/framework/lod_rank_table.cc @@ -0,0 +1,48 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/lod_rank_table.h" + +namespace paddle { +namespace framework { +void LoDRankTable::Reset(const LoD& lod, size_t level) { + this->coarse_lod_.clear(); + this->items_.clear(); + PADDLE_ENFORCE(level < lod.size(), + "Cannot rank lod since the level %d is less than lod size %d", + level, lod.size()); + coarse_lod_.reserve(level); + for (size_t i = 0; i < level; ++i) { + coarse_lod_.push_back(lod[i]); + } + auto& vec = lod[level]; + for (size_t i = 0; i < vec.size() - 1; ++i) { + TableItem item; + item.index = i; + item.length = vec[i + 1] - vec[i]; + items_.emplace_back(item); + } + // NOTE(yuyang18): + // + // The time complexity of stable_sort is O(N*log(N)) if additional memory is + // available. It is easy to debug and unit test when using `stable_sort` + // instead of `sort`. Also, the items of a rank table will not be too large. + std::stable_sort(items_.begin(), items_.end(), + [](const TableItem& a, const TableItem& b) { + return a.length > b.length; + }); +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_rank_table.h b/paddle/framework/lod_rank_table.h new file mode 100644 index 0000000000000000000000000000000000000000..9faa3a4d7bdc55ab7b24e31f5e5434dacc0a4b36 --- /dev/null +++ b/paddle/framework/lod_rank_table.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { + +// LoD Rank Table stores the `level` of `lod` which is ordered by sequence +// length in descending order. It is useful when implement dynamic RNN and is +// shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +// output operators. +// +// The table item contains two element. The length of sequence and the index of +// sequence in that level. +// +// LoDRankTable also stores the coarse_lod, which is the lod information whose +// level is less than input level, in order to restore the output LoD +// information. +class LoDRankTable { + public: + struct TableItem { + size_t index; + size_t length; + }; + + LoDRankTable() {} + + void Reset(const LoD& lod, size_t level); + + const std::vector& items() const { return this->items_; } + + const LoD& coarse_lod() const { return this->coarse_lod_; } + + size_t level() const { return coarse_lod_.size(); } + + private: + LoD coarse_lod_; + std::vector items_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 584308a5388da0d02d29f71a28097b02b6ea825f..2bcfffb134f46416301b28043e1875e822fbc3e4 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -135,5 +135,43 @@ void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin, PADDLE_ENFORCE_LT(begin, end, "Cannot shrink, the result tensor is empty."); ShareDataWith(Slice(begin, end)); } + +void GetFineGrainedLoDLength(const LoD& lod, size_t start_idx, size_t end_idx, + std::vector>* lod_length, + size_t* start_offset) { + lod_length->clear(); + PADDLE_ENFORCE(start_idx < lod.size() - 1, + "start_idx should be >= 0 and < lod.size() - 1."); + PADDLE_ENFORCE(end_idx < lod.size(), + "end_idx should be >= 0 and < lod.size()."); + PADDLE_ENFORCE_LE(start_idx, end_idx, + "start_idx should be less than end_idx."); + for (size_t level_idx = 0; level_idx < lod.size(); ++level_idx) { + std::vector level_lens; + for (size_t i = start_idx; i < end_idx; ++i) { + level_lens.push_back(lod[level_idx][i + 1] - lod[level_idx][i]); + } + lod_length->emplace_back(level_lens); + start_idx = lod[level_idx][start_idx]; + end_idx = lod[level_idx][end_idx]; + } + *start_offset = start_idx; +} + +void AppendLoD(LoD* lod, const std::vector>& lod_length) { + PADDLE_ENFORCE_EQ( + lod->size(), lod_length.size(), + "The lod_length should has the same size with the appended lod."); + for (size_t i = 0; i < lod->size(); ++i) { + auto& level = (*lod)[i]; + if (level.empty()) { + level.push_back(0); + } + for (size_t len : lod_length[i]) { + level.push_back(level.back() + len); + } + } +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index f4fe4cdac6019a1899fd3db8e1b6ca588be0d436..1437da399a28288429527f9672ace0482825159f 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -181,5 +181,11 @@ LoDTensor LodExpand(const LoDTensor& source, const LoD& lod, size_t level, return tensor; } +void GetFineGrainedLoDLength(const LoD& lod, size_t start_idx, size_t end_idx, + std::vector>* lod_length, + size_t* start_offset); + +void AppendLoD(LoD* lod, const std::vector>& lod_length); + } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor_array.h b/paddle/framework/lod_tensor_array.h new file mode 100644 index 0000000000000000000000000000000000000000..13f0608d24be97d8bba149b74f1a4deb57deeb48 --- /dev/null +++ b/paddle/framework/lod_tensor_array.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { +using LoDTensorArray = std::vector; +} +} // namespace paddle diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index aa2f6c993d41ae98e0769d470dccad3b410da53e..bf61c9ee7aa99b06e78b9b27dff72d216c74a86e 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -144,5 +144,47 @@ TEST(LodExpand, test) { } } +TEST(LoD, GetFineGrainedLoDLength) { + LoD lod; + lod.push_back(std::vector{0, 2, 4, 5}); + lod.push_back(std::vector{0, 1, 6, 8, 10, 11}); + lod.push_back( + std::vector{0, 2, 5, 7, 10, 12, 15, 17, 20, 24, 26, 29}); + + std::vector> lod_length; + size_t start_offset; + paddle::framework::GetFineGrainedLoDLength(lod, 1, 2, &lod_length, + &start_offset); + + std::vector> expected; + expected.push_back(std::vector{2}); + expected.push_back(std::vector{2, 2}); + expected.push_back(std::vector{2, 3, 4, 2}); + EXPECT_EQ(lod_length, expected); + EXPECT_EQ(start_offset, 15UL); +} + +TEST(LoD, AppendLoD) { + std::vector> lod_lens; + lod_lens.push_back(std::vector{2}); + lod_lens.push_back(std::vector{2, 2}); + lod_lens.push_back(std::vector{2, 3, 4, 2}); + + LoD origin; + origin.push_back(std::vector{0, 2}); + origin.push_back(std::vector{0, 1, 6}); + origin.push_back(std::vector{0, 2, 5, 7, 10, 12, 15}); + + paddle::framework::AppendLoD(&origin, lod_lens); + + LoD expected; + expected.push_back(std::vector{0, 2, 4}); + expected.push_back(std::vector{0, 1, 6, 8, 10}); + expected.push_back( + std::vector{0, 2, 5, 7, 10, 12, 15, 17, 20, 24, 26}); + + EXPECT_EQ(origin, expected); +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor_test.cu b/paddle/framework/lod_tensor_test.cu index c79c4d0c721f9e568c937cb9e524e925fcdc83d0..5b90fbfca7f6bec4f2c862d0ff18dfd7cf39e181 100644 --- a/paddle/framework/lod_tensor_test.cu +++ b/paddle/framework/lod_tensor_test.cu @@ -36,8 +36,8 @@ TEST(LoDTensor, LoDInGPU) { lod_tensor.mutable_data(place); lod_tensor.set_lod(src_lod); - CHECK_EQ(lod_tensor.lod_element(0, 2).first, 4UL); - CHECK_EQ(lod_tensor.lod_element(0, 4).first, 8UL); + EXPECT_EQ(lod_tensor.lod_element(0, 2).first, 4UL); + EXPECT_EQ(lod_tensor.lod_element(0, 4).first, 8UL); auto lod = lod_tensor.lod(); @@ -45,6 +45,6 @@ TEST(LoDTensor, LoDInGPU) { cudaDeviceSynchronize(); for (size_t i = 0; i < src_lod[0].size(); ++i) { - CHECK_EQ(lod[0].data()[i], src_lod[0].data()[i] * 2); + EXPECT_EQ(lod[0].data()[i], src_lod[0].data()[i] * 2); } -} \ No newline at end of file +} diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index c2d6f124ad292bf46b4e7e9a1dcc2984aae7fcda..c96166f35d1425218a4a74f50dc5ed542d677b68 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -52,6 +52,22 @@ class CompileTimeInferShapeContext : public InferShapeContext { const std::vector &Outputs( const std::string &name) const override; + void ShareLoD(const std::string &in, const std::string &out, size_t i = 0, + size_t j = 0) const override { + PADDLE_ENFORCE_LT(i, Inputs(in).size()); + PADDLE_ENFORCE_LT(j, Outputs(out).size()); + auto *in_var = block_.FindVarRecursive(Inputs(in)[i]); + auto *out_var = block_.FindVarRecursive(Outputs(out)[j]); + if (in_var->GetType() != VarDesc::LOD_TENSOR) { + VLOG(3) << "input " << in << "is not LodTensor"; + return; + } + PADDLE_ENFORCE_EQ(in_var->GetType(), VarDesc::LOD_TENSOR, + "The %d-th output of Output(%s) must be LoDTensor.", j, + out); + in_var->SetLoDLevel(out_var->GetLodLevel()); + } + private: DDim GetDim(const std::string &name) const override; @@ -98,7 +114,12 @@ OpDescBind::OpDescBind(const OpDesc &desc, ProgramDescBind *prog) // restore attrs_ for (const OpDesc::Attr &attr : desc_.attrs()) { std::string attr_name = attr.name(); - attrs_[attr_name] = GetAttrValue(attr, prog->Proto()); + if (attr.type() != AttrType::BLOCK) { + attrs_[attr_name] = GetAttrValue(attr); + } else { + auto bid = attr.block_idx(); + attrs_[attr_name] = prog->MutableBlock(bid); + } } } @@ -172,8 +193,7 @@ void OpDescBind::SetAttr(const std::string &name, const Attribute &v) { } void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) { - BlockDesc *desc = block.Proto(); - this->attrs_[name] = desc; + this->attrs_[name] = █ need_update_ = true; } @@ -192,7 +212,7 @@ Attribute OpDescBind::GetAttr(const std::string &name) const { int OpDescBind::GetBlockAttr(const std::string &name) const { auto it = attrs_.find(name); PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); - return boost::get(it->second)->idx(); + return boost::get(it->second)->ID(); } const std::unordered_map &OpDescBind::GetAttrMap() @@ -307,6 +327,19 @@ void OpDescBind::InferShape(const BlockDescBind &block) const { PADDLE_ENFORCE(static_cast(infer_shape), "%s's infer_shape has not been registered", this->Type()); CompileTimeInferShapeContext ctx(*this, block); + if (VLOG_IS_ON(10)) { + std::ostringstream sout; + auto inames = this->InputArgumentNames(); + sout << " From ["; + std::copy(inames.begin(), inames.end(), + std::ostream_iterator(sout, ", ")); + sout << "] to ["; + auto onames = this->OutputArgumentNames(); + std::copy(onames.begin(), onames.end(), + std::ostream_iterator(sout, ", ")); + sout << "]"; + VLOG(10) << sout.str(); + } infer_shape(&ctx); } diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index c2f2438edf6daadf26cbc6db37f6668739ab1726..8dedd873aad648174b770b84e5232cd17b577e72 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -43,13 +43,15 @@ static VariableNameMap ConvertOpDescVarsToVarNameMap( return ret_val; } -std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc, - ProgramDesc* program) { +std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { + VLOG(1) << "CreateOp directly from OpDesc is deprecated. It should only be" + "used in unit tests. Use CreateOp(const OpDescBind& op_desc) " + "instead."; VariableNameMap inputs = ConvertOpDescVarsToVarNameMap(op_desc.inputs()); VariableNameMap outputs = ConvertOpDescVarsToVarNameMap(op_desc.outputs()); AttributeMap attrs; for (auto& attr : op_desc.attrs()) { - attrs[attr.name()] = GetAttrValue(attr, program); + attrs[attr.name()] = GetAttrValue(attr); } return CreateOp(op_desc.type(), inputs, outputs, attrs); diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 19a9fc3802a2f2348ad7d50a267615ed70bbc4fe..2bb5e0e8ec29fb2df81549650aa0c65bc1e51c49 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -77,8 +77,7 @@ class OpRegistry { const VariableNameMap& outputs, AttributeMap attrs); - static std::unique_ptr CreateOp(const OpDesc& op_desc, - ProgramDesc* program); + static std::unique_ptr CreateOp(const OpDesc& op_desc); static std::unique_ptr CreateOp(const OpDescBind& op_desc); }; diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index 6289125d7c782e542e5c55e1d4403836351b7e05..b860fe6cac773d1e85adecc43f5dfec42b6c7661 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -74,7 +74,7 @@ TEST(OpRegistry, CreateOp) { attr->set_type(paddle::framework::AttrType::FLOAT); attr->set_f(scale); - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); paddle::framework::Scope scope; paddle::platform::CPUDeviceContext dev_ctx; op->Run(scope, dev_ctx); @@ -95,7 +95,7 @@ TEST(OpRegistry, IllegalAttr) { bool caught = false; try { - paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::platform::EnforceNotMet err) { caught = true; std::string msg = "larger_than check fail"; @@ -115,7 +115,7 @@ TEST(OpRegistry, DefaultValue) { ASSERT_TRUE(op_desc.IsInitialized()); - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); paddle::framework::Scope scope; paddle::platform::CPUDeviceContext dev_ctx; op->Run(scope, dev_ctx); @@ -131,7 +131,7 @@ TEST(OpRegistry, CustomChecker) { // attr 'test_attr' is not set bool caught = false; try { - paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::platform::EnforceNotMet err) { caught = true; std::string msg = "Attribute 'test_attr' is required!"; @@ -149,7 +149,7 @@ TEST(OpRegistry, CustomChecker) { attr->set_i(3); caught = false; try { - paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::platform::EnforceNotMet err) { caught = true; std::string msg = "'test_attr' must be even!"; @@ -166,7 +166,7 @@ TEST(OpRegistry, CustomChecker) { attr->set_name("test_attr"); attr->set_type(paddle::framework::AttrType::INT); attr->set_i(4); - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); paddle::platform::CPUDeviceContext dev_ctx; paddle::framework::Scope scope; op->Run(scope, dev_ctx); diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 222a252dc409bf30d5d6abea95156b41cfcd221a..9295d36c2b2e66130ad273ebd3a40de739efeea7 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -37,32 +37,32 @@ ExecutionContext::GetEigenDevice() const { std::string OperatorBase::Input(const std::string& name) const { auto& ins = Inputs(name); PADDLE_ENFORCE_LE(ins.size(), 1UL, - "Op %s input %s should contain only one variable", type_, - name); + "Operator %s's input %s should contain only one variable.", + type_, name); return ins.empty() ? kEmptyVarName : ins[0]; } const std::vector& OperatorBase::Inputs( const std::string& name) const { auto it = inputs_.find(name); - PADDLE_ENFORCE(it != inputs_.end(), "Op %s do not have input %s", type_, - name); + PADDLE_ENFORCE(it != inputs_.end(), "Operator %s does not have the input %s.", + type_, name); return it->second; } std::string OperatorBase::Output(const std::string& name) const { auto& outs = Outputs(name); PADDLE_ENFORCE_LE(outs.size(), 1UL, - "Op %s output %s should contain only one variable", type_, - name); + "Operator %s's output %s should contain only one variable.", + type_, name); return outs.empty() ? kEmptyVarName : outs[0]; } const std::vector& OperatorBase::Outputs( const std::string& name) const { auto it = outputs_.find(name); - PADDLE_ENFORCE(it != outputs_.end(), "Op %s does not have output called %s", - type_, name); + PADDLE_ENFORCE(it != outputs_.end(), + "Operator %s does not have an output called %s.", type_, name); return it->second; } @@ -126,7 +126,7 @@ OperatorBase::OperatorBase(const std::string& type, std::vector OperatorBase::InputVars() const { std::vector ret_val; - for (auto& o : outputs_) { + for (auto& o : inputs_) { ret_val.reserve(ret_val.size() + o.second.size()); ret_val.insert(ret_val.end(), o.second.begin(), o.second.end()); } @@ -351,6 +351,20 @@ class RuntimeInferShapeContext : public InferShapeContext { return op_.Outputs(name); } + void ShareLoD(const std::string& in, const std::string& out, size_t i = 0, + size_t j = 0) const override { + PADDLE_ENFORCE_LT(i, Inputs(in).size()); + PADDLE_ENFORCE_LT(j, Outputs(out).size()); + Variable* in_var = scope_.FindVar(Inputs(in)[i]); + Variable* out_var = scope_.FindVar(Outputs(out)[j]); + if (!in_var->IsType()) return; + PADDLE_ENFORCE(out_var->IsType(), + "The %d-th output of Output(%s) must be LoDTensor.", j, out); + auto in_tensor = in_var->Get(); + auto* out_tensor = out_var->GetMutable(); + out_tensor->set_lod(in_tensor.lod()); + } + private: DDim GetDim(const std::string& name) const override { Variable* var = scope_.FindVar(name); @@ -380,7 +394,19 @@ class RuntimeInferShapeContext : public InferShapeContext { void OperatorWithKernel::Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const { - VLOG(3) << "Running operator " << this->Type(); + if (VLOG_IS_ON(1)) { + auto inputs = this->InputVars(); + auto outputs = this->OutputVars(true); + std::ostringstream sout; + sout << "Run operator " << this->Type() << " From ["; + std::ostream_iterator out_it(sout, ","); + std::copy(inputs.begin(), inputs.end(), out_it); + sout << "] to ["; + std::copy(outputs.begin(), outputs.end(), out_it); + sout << "]"; + VLOG(1) << sout.str(); + } + RuntimeInferShapeContext infer_shape_ctx(*this, scope); this->InferShape(&infer_shape_ctx); diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 93885fa3028e072bc0bd021ea9287087678f3621..5c1989c26b68413ea52b97128313609e5d917fd1 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -408,7 +408,6 @@ class OperatorWithKernel : public OperatorBase { // indicate kernel DataType by input data. Defaultly all input data must be // same. virtual DataType IndicateDataType(const ExecutionContext& ctx) const { - VLOG(3) << "Default IndicateDataType " << this->Type(); auto& scope = ctx.scope(); int data_type = -1; for (auto& input : this->inputs_) { @@ -425,9 +424,9 @@ class OperatorWithKernel : public OperatorBase { } if (t != nullptr) { int tmp = static_cast(ToDataType(t->type())); - VLOG(3) << "Input " << ipt_name << " with data_type " << tmp; PADDLE_ENFORCE(tmp == data_type || data_type == -1, - "DataType of Paddle Op %s must be same.", Type()); + "DataType of Paddle Op %s must be the same.", + Type()); data_type = tmp; } } diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 3c07621293389fc7803b0295d9d30b2c12d6e327..42e0d52eed3911d8e684e76a88bc690ca0783ce5 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -83,7 +83,7 @@ TEST(OperatorBase, all) { paddle::platform::CPUDeviceContext device_context; paddle::framework::Scope scope; - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); scope.Var("OUT1"); ASSERT_EQ(paddle::framework::op_run_num, 0); op->Run(scope, device_context); @@ -208,7 +208,7 @@ TEST(OpKernel, all) { paddle::platform::CPUDeviceContext cpu_device_context; paddle::framework::Scope scope; - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); ASSERT_EQ(paddle::framework::cpu_kernel_run_num, 0); op->Run(scope, cpu_device_context); ASSERT_EQ(paddle::framework::cpu_kernel_run_num, 1); @@ -244,7 +244,7 @@ TEST(OpKernel, multi_inputs) { scope.Var("y0")->GetMutable(); scope.Var("y1")->GetMutable(); - auto op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + auto op = paddle::framework::OpRegistry::CreateOp(op_desc); op->Run(scope, cpu_device_context); } diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h index ce1721472d9046f50b7fc88253fa3f2dbaaf51a8..b1cb086de4345902482d8254b8aeec041ecf81bc 100644 --- a/paddle/framework/program_desc.h +++ b/paddle/framework/program_desc.h @@ -37,7 +37,9 @@ class ProgramDescBind { BlockDescBind *AppendBlock(const BlockDescBind &parent); - BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); } + BlockDescBind *MutableBlock(size_t idx) { return blocks_[idx].get(); } + + const BlockDescBind &Block(size_t idx) const { return *blocks_[idx]; } size_t Size() const { return blocks_.size(); } diff --git a/paddle/framework/program_desc_test.cc b/paddle/framework/program_desc_test.cc index d28c2a0bff932f5aa37c69231495895dacb07bb3..83e7286e0ec3639fa589b0958922543a3ba16a00 100644 --- a/paddle/framework/program_desc_test.cc +++ b/paddle/framework/program_desc_test.cc @@ -20,7 +20,7 @@ namespace paddle { namespace framework { TEST(ProgramDesc, copy_ctor) { ProgramDescBind program; - auto* global_block = program.Block(0); + auto* global_block = program.MutableBlock(0); auto* x = global_block->Var("X"); x->SetType(VarDesc_VarType_LOD_TENSOR); x->SetLoDLevel(0); @@ -44,7 +44,7 @@ TEST(ProgramDesc, copy_ctor) { ProgramDescBind program_copy(program); - auto* global_block_copy = program_copy.Block(0); + auto* global_block_copy = program_copy.MutableBlock(0); ASSERT_NE(global_block, global_block_copy); auto assert_same_var = [&](const std::string& name, VarDescBind* var_before) { @@ -82,7 +82,7 @@ TEST(ProgramDesc, copy_ctor) { TEST(ProgramDescBind, serialize_and_deserialize) { ProgramDescBind program_origin; - auto* global_block = program_origin.Block(0); + auto* global_block = program_origin.MutableBlock(0); auto* x = global_block->Var("X"); x->SetType(VarDesc_VarType_LOD_TENSOR); x->SetLoDLevel(0); @@ -108,7 +108,7 @@ TEST(ProgramDescBind, serialize_and_deserialize) { program_origin.Proto()->SerializeToString(&binary_str); ProgramDescBind program_restored(binary_str); - auto* global_block_restored = program_restored.Block(0); + auto* global_block_restored = program_restored.MutableBlock(0); ASSERT_NE(global_block, global_block_restored); auto assert_same_var = [&](const std::string& name, VarDescBind* var_before) { diff --git a/paddle/framework/prune_test.cc b/paddle/framework/prune_test.cc index cadd114fbc3de897a13504e665ce464e83d312ff..5988874809f51c09b3d3d279be6c1e8d43d7a782 100644 --- a/paddle/framework/prune_test.cc +++ b/paddle/framework/prune_test.cc @@ -52,7 +52,7 @@ void AddOp(const std::string &type, const f::VariableNameMap &inputs, TEST(Prune, one_operator) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a"}}}, {{"output", {"b"}}}, {}, block); @@ -69,7 +69,7 @@ TEST(Prune, one_operator) { TEST(Prune, forward) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a"}}}, {{"output", {"b"}}}, {}, block); AddOp("one_one", {{"input", {"b"}}}, {{"output", {"c"}}}, {}, block); @@ -88,7 +88,7 @@ TEST(Prune, forward) { TEST(Prune, multi_input_op) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); AddOp("one_one", {{"input", {"a0"}}}, {{"output", {"b0"}}}, {}, block); AddOp("one_one", {{"input", {"a1"}}}, {{"output", {"b1"}}}, {}, block); @@ -106,7 +106,7 @@ TEST(Prune, multi_input_op) { TEST(Prune, multi_output_op) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); AddOp("one_two", {{"input", {"a"}}}, {{"output", {"b", "c"}}}, {}, block); AddOp("one_one", {{"input", {"b"}}}, {{"output", {"b1"}}}, {}, block); @@ -122,7 +122,7 @@ TEST(Prune, multi_output_op) { TEST(Prune, multi_target) { f::ProgramDescBind program; - f::BlockDescBind *block = program.Block(0); + f::BlockDescBind *block = program.MutableBlock(0); AddOp("one_two", {{"input", {"a"}}}, {{"output", {"b", "c"}}}, {}, block); AddOp("one_one", {{"input", {"b"}}}, {{"output", {"b1"}}}, {}, block); diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc index 14cc530448379eb6d4bf0435f607494aa01ef5b5..fb2c69105627f663ddcce07d31526c9e4278e863 100644 --- a/paddle/framework/scope.cc +++ b/paddle/framework/scope.cc @@ -47,8 +47,12 @@ Variable* Scope::Var(const std::string& name) { return v; } -Variable* Scope::Var() { - return Var(string::Sprintf("%p.%d", this, vars_.size())); +Variable* Scope::Var(std::string* name) { + auto var_name = string::Sprintf("%p.%d", this, vars_.size()); + if (name != nullptr) { + *name = var_name; + } + return Var(var_name); } Variable* Scope::FindVar(const std::string& name) const { diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index ac334da5ef0c8ad563b6be5413df33f5d0bdbcf8..fb660949394149ebf2c6172a0ac3f4c7594f4286 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -49,7 +49,7 @@ class Scope { Variable* Var(const std::string& name); /// Create a variable with a scope-unique name. - Variable* Var(); + Variable* Var(std::string* name = nullptr); /// Find a variable in the scope or any of its ancestors. Returns /// nullptr if cannot find. diff --git a/paddle/framework/shape_inference.cc b/paddle/framework/shape_inference.cc index 33a1d0b9b217c5d2a4b0fb63f427529e7988b24e..8169df8e4629e2d02d3dabcd6a8a102ad0077a81 100644 --- a/paddle/framework/shape_inference.cc +++ b/paddle/framework/shape_inference.cc @@ -28,9 +28,6 @@ void InferShapeContext::SetOutputsDim( SetDims(names, dims); } -void InferShapeContext::ShareLoD(const std::string &in, const std::string &out, - size_t i, size_t j) const {} - std::vector InferShapeContext::GetDims( const std::vector &names) const { std::vector ret; diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index f1f1e44bccd771be81cad7c28efe9b1b885eef6b..6f19900ef1a3e88fe78d457a03c344ea586ab551 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -43,9 +43,8 @@ class InferShapeContext { virtual const std::vector &Outputs( const std::string &name) const = 0; - // TODO(qiao) implement this function - void ShareLoD(const std::string &in, const std::string &out, size_t i = 0, - size_t j = 0) const; + virtual void ShareLoD(const std::string &in, const std::string &out, + size_t i = 0, size_t j = 0) const = 0; protected: virtual framework::DDim GetDim(const std::string &name) const = 0; diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h index 7b9a5b75e1087a1cc3b6c6c7a6e4dc185c32dd42..28d0fcf94ec31c82476e093f93ccee222a0c9d9a 100644 --- a/paddle/framework/tensor.h +++ b/paddle/framework/tensor.h @@ -118,12 +118,14 @@ class Tensor { const platform::DeviceContext& ctx); /** - * @brief Return the slice of the tensor. + * @brief Return a sub-tensor of the given tensor. * - * @param[in] begin_idx The begin index of the slice. - * @param[in] end_idx The end index of the slice. + * @param[in] begin_idx The index of the start row(inclusive) to slice. + * The index number begins from 0. + * @param[in] end_idx The index of the end row(exclusive) to slice. + * The index number begins from 0. */ - inline Tensor Slice(const int& begin_idx, const int& end_idx) const; + inline Tensor Slice(int begin_idx, int end_idx) const; platform::Place place() const { PADDLE_ENFORCE_NOT_NULL( diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index 29ac683f48fcde4dd3b5ad7f04b5d1d7434706ba..d78a2c4c21149ef3c800991b9a144ea198f1bdcf 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -112,9 +112,10 @@ inline void* Tensor::mutable_data(platform::Place place, std::type_index type) { if (holder_ != nullptr) { holder_->set_type(type); } - PADDLE_ENFORCE_GT(numel(), 0, - "Tensor's numel must be larger than zero to call " - "Tensor::mutable_data. Call Tensor::set_dim first."); + PADDLE_ENFORCE_GT( + numel(), 0, + "When calling this method, the Tensor's numel must be larger than zero. " + "Please check Tensor::Resize has been called first."); int64_t size = numel() * SizeOfType(type); /* some versions of boost::variant don't have operator!= */ if (holder_ == nullptr || !(holder_->place() == place) || @@ -227,12 +228,14 @@ inline void Tensor::CopyFromVector(const std::vector& src, #endif } -inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const { +inline Tensor Tensor::Slice(int begin_idx, int end_idx) const { check_memory_size(); - PADDLE_ENFORCE_GE(begin_idx, 0, "Slice begin index is less than zero."); - PADDLE_ENFORCE_LE(end_idx, dims_[0], "Slice end index is out of bound."); - PADDLE_ENFORCE_LT(begin_idx, end_idx, - "Begin index must be less than end index."); + PADDLE_ENFORCE_GE(begin_idx, 0, + "The start row index must be greater than 0."); + PADDLE_ENFORCE_LE(end_idx, dims_[0], "The end row index is out of bound."); + PADDLE_ENFORCE_LT( + begin_idx, end_idx, + "The start row index must be lesser than the end row index."); if (dims_[0] == 1) { return *this; diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index c38c4a8ae9a46c8bda913e7643e812592de68e6e..baeb98c9bd49ec65da5931bcbe33ab788f86f3e8 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -29,6 +29,7 @@ class OpDescBind; class BlockDescBind; class BlockDesc; class InferShapeContext; +class BlockDescBind; using VariableNameMap = std::map>; @@ -36,7 +37,7 @@ using VariableNameMap = std::map>; using Attribute = boost::variant, std::vector, std::vector, bool, - std::vector, BlockDesc*>; + std::vector, BlockDescBind*>; using AttributeMap = std::unordered_map; @@ -46,7 +47,8 @@ using OpCreator = std::function>( const OpDescBind&, const std::unordered_set& /*no_grad_set*/, - std::unordered_map* /*grad_to_var*/)>; + std::unordered_map* /*grad_to_var*/, + const std::vector& grad_block)>; using InferVarTypeFN = std::function; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 8e92c81d1137472737230be79d71824593d3256f..16aca192d41a64003de85ce45f7697bf45c556ed 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -37,13 +37,27 @@ std::vector VarDescBind::Shape() const { DataType VarDescBind::GetDataType() const { return tensor_desc().data_type(); } void VarDescBind::SetLoDLevel(int32_t lod_level) { - PADDLE_ENFORCE(desc_.type() == VarDesc::LOD_TENSOR); - desc_.mutable_lod_tensor()->set_lod_level(lod_level); + switch (desc_.type()) { + case VarDesc::LOD_TENSOR: + desc_.mutable_lod_tensor()->set_lod_level(lod_level); + break; + case VarDesc::LOD_TENSOR_ARRAY: + desc_.mutable_tensor_array()->set_lod_level(lod_level); + break; + default: + PADDLE_THROW("Tensor type=%d does not support LoDLevel", desc_.type()); + } } int32_t VarDescBind::GetLodLevel() const { - PADDLE_ENFORCE(desc_.type() == VarDesc::LOD_TENSOR); - return desc_.lod_tensor().lod_level(); + switch (desc_.type()) { + case VarDesc::LOD_TENSOR: + return desc_.lod_tensor().lod_level(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.tensor_array().lod_level(); + default: + PADDLE_THROW("Tensor type=%d does not support LoDLevel", desc_.type()); + } } const TensorDesc &VarDescBind::tensor_desc() const { @@ -53,6 +67,8 @@ const TensorDesc &VarDescBind::tensor_desc() const { return desc_.selected_rows(); case VarDesc::LOD_TENSOR: return desc_.lod_tensor().tensor(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.tensor_array().tensor(); default: PADDLE_THROW("Unexpected branch."); } @@ -66,6 +82,8 @@ TensorDesc *VarDescBind::mutable_tensor_desc() { return desc_.mutable_selected_rows(); case VarDesc::LOD_TENSOR: return desc_.mutable_lod_tensor()->mutable_tensor(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.mutable_tensor_array()->mutable_tensor(); default: PADDLE_THROW("Unexpected branch."); } diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h index 70daa20e8d99abc5759655adf538a8c197e9ec6a..5cf4608944c5011d798fbde060002a57be8f6102 100644 --- a/paddle/framework/var_desc.h +++ b/paddle/framework/var_desc.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include "glog/logging.h" #include "paddle/framework/framework.pb.h" namespace paddle { diff --git a/paddle/framework/var_type_inference_test.cc b/paddle/framework/var_type_inference_test.cc index 918de1fd055e32888f71ffea1f33993ba1210e86..9035e63fa48ffdf7c72061b0a4248538d7a357e4 100644 --- a/paddle/framework/var_type_inference_test.cc +++ b/paddle/framework/var_type_inference_test.cc @@ -63,41 +63,43 @@ namespace framework { TEST(InferVarType, sum_op) { ProgramDescBind prog; - auto *op = prog.Block(0)->AppendOp(); + auto *op = prog.MutableBlock(0)->AppendOp(); op->SetType("sum"); op->SetInput("X", {"test_a", "test_b", "test_c"}); op->SetOutput("Out", {"test_out"}); - prog.Block(0)->Var("test_a")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test_b")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test_c")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test_out"); + prog.MutableBlock(0)->Var("test_a")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test_b")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test_c")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test_out"); - op->InferVarType(prog.Block(0)); + op->InferVarType(prog.MutableBlock(0)); - ASSERT_EQ(VarDesc::SELECTED_ROWS, prog.Block(0)->Var("test_out")->GetType()); + ASSERT_EQ(VarDesc::SELECTED_ROWS, + prog.MutableBlock(0)->Var("test_out")->GetType()); - prog.Block(0)->Var("test_b")->SetType(VarDesc::LOD_TENSOR); - op->InferVarType(prog.Block(0)); - ASSERT_EQ(VarDesc::LOD_TENSOR, prog.Block(0)->Var("test_out")->GetType()); + prog.MutableBlock(0)->Var("test_b")->SetType(VarDesc::LOD_TENSOR); + op->InferVarType(prog.MutableBlock(0)); + ASSERT_EQ(VarDesc::LOD_TENSOR, + prog.MutableBlock(0)->Var("test_out")->GetType()); } TEST(InferVarType, sum_op_without_infer_var_type) { ProgramDescBind prog; - auto *op = prog.Block(0)->AppendOp(); + auto *op = prog.MutableBlock(0)->AppendOp(); op->SetType("sum_without_infer_var_type"); op->SetInput("X", {"test2_a", "test2_b", "test2_c"}); op->SetOutput("Out", {"test2_out"}); - prog.Block(0)->Var("test2_a")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test2_b")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test2_c")->SetType(VarDesc::SELECTED_ROWS); - prog.Block(0)->Var("test2_out"); + prog.MutableBlock(0)->Var("test2_a")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test2_b")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test2_c")->SetType(VarDesc::SELECTED_ROWS); + prog.MutableBlock(0)->Var("test2_out"); - op->InferVarType(prog.Block(0)); + op->InferVarType(prog.MutableBlock(0)); ASSERT_EQ(VarDesc_VarType_LOD_TENSOR, - prog.Block(0)->Var("test2_out")->GetType()); + prog.MutableBlock(0)->Var("test2_out")->GetType()); } } // namespace framework diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 9db6d252d97bfeee3fe376bcda431fe94c65a678..8e66b1f0db5d8a365a5aa9b98d2fb3f867458411 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -395,14 +395,24 @@ real AucEvaluator::evalImp(std::vector& arguments) { CHECK_LE(arguments.size(), (size_t)3); MatrixPtr output = arguments[0].value; IVectorPtr label = arguments[1].ids; + MatrixPtr labelval = arguments[1].value; bool supportWeight = (3 == arguments.size()) ? true : false; MatrixPtr weight = supportWeight ? arguments[2].value : nullptr; - if (nullptr == output || nullptr == label || - (supportWeight && nullptr == weight)) { + + if (nullptr == output || (supportWeight && nullptr == weight)) { return 0; } size_t insNum = output->getHeight(); size_t outputDim = output->getWidth(); + // Copy label from value to a vector. + if (nullptr == label && nullptr != labelval) { + // label width is 1 + CHECK_EQ(1U, labelval->getWidth()); + VectorPtr vec = + Vector::create(labelval->getData(), insNum, output->useGpu()); + label = vec->castToInt(); + } + CHECK_EQ(insNum, label->getSize()); if (supportWeight) { CHECK_EQ(insNum, weight->getHeight()); @@ -443,6 +453,7 @@ real AucEvaluator::evalImp(std::vector& arguments) { int* labelD = label->getData(); real* weightD = supportWeight ? weight->getData() : nullptr; size_t pos = realColumnIdx_; + for (size_t i = 0; i < insNum; ++i) { real value = outputD[pos]; uint32_t binIdx = static_cast(value * kBinNum_); diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index 0b544420097e9150f8489731b6379dea633e992c..867303b4fa0d490297ab152fc2ad266e92e29baf 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -101,8 +101,10 @@ void CRFLayer::backward(const UpdateCallback& callback) { : real(1.0f); instanceWeight *= coeff_; - MatrixPtr grad = output.grad->subRowMatrix(starts[i], starts[i + 1]); - grad->add(*crfs_[i].getXGrad(), real(1.0f), instanceWeight); + if (output.grad) { + MatrixPtr grad = output.grad->subRowMatrix(starts[i], starts[i + 1]); + grad->add(*crfs_[i].getXGrad(), real(1.0f), instanceWeight); + } if (needWGrad) { weight_->getWGrad()->add( *crfs_[i].getWGrad(), real(1.0f), instanceWeight); diff --git a/paddle/gserver/layers/LinearChainCRF.cpp b/paddle/gserver/layers/LinearChainCRF.cpp index dc3dc156792bdf32c3b948a292597d0e9eca5d8b..abaa1802b763a49f748214dbd4dec1d2bac53b59 100644 --- a/paddle/gserver/layers/LinearChainCRF.cpp +++ b/paddle/gserver/layers/LinearChainCRF.cpp @@ -102,7 +102,6 @@ real LinearChainCRF::forward(real* x, int* s, int length) { } void LinearChainCRF::backward(real* x, int* s, int length, bool needWGrad) { - MatrixPtr matX = Matrix::create(x, length, numClasses_); Matrix::resizeOrCreate(matGrad_, length, numClasses_); Matrix::resizeOrCreate(beta_, length, numClasses_); real* b = b_->getData(); diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.cpp b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8eb700723f2cf7dda969739bb5e3d48358d278a0 --- /dev/null +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp @@ -0,0 +1,154 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MKLDNNAddtoLayer.h" + +using namespace mkldnn; // NOLINT + +namespace paddle { + +REGISTER_LAYER(mkldnn_addto, MKLDNNAddtoLayer); + +bool MKLDNNAddtoLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + if (!MKLDNNLayer::init(layerMap, parameterMap)) { + return false; + } + + layerSize_ = getSize(); + for (size_t i = 0; i < inputLayers_.size(); i++) { + CHECK_EQ(layerSize_, inputLayers_[i]->getSize()) << "input size must equal"; + } + if (biasParameter_.get() != NULL) { + biases_ = + std::unique_ptr(new Weight(1, layerSize_, biasParameter_, 0)); + } + return true; +} + +void MKLDNNAddtoLayer::reshape( + int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) { + CHECK_EQ(layerSize_, getSize()) << "this layer size can not be changed"; + reshapeInput(bs, ih, iw); + ic = inputLayers_[0]->getSize() / ih / iw; + CHECK_EQ((size_t)ic * ih * iw, inputLayers_[0]->getSize()); + CHECK_EQ(inputElemenCnt_, (size_t)bs * ic * ih * iw); + for (size_t i = 0; i < inputLayers_.size(); i++) { + CHECK_EQ(int64_t(bs), inputLayers_[i]->getOutput().getBatchSize()); + CHECK_EQ(layerSize_, inputLayers_[i]->getSize()); + } + + oc = ic; + oh = ih; + ow = iw; + reshapeOutput(oh, ow); + resizeOutput(bs, oc * oh * ow); + printSizeInfo(); +} + +void MKLDNNAddtoLayer::resetFwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + if (biases_) { + LOG(FATAL) << "not implemented yet"; + } + resetFwdBuffers(inVals_, out); + in = inVals_[0]; + + std::shared_ptr fwdPD; + resetFwdPD(fwdPD, inVals_, out); + + resetFwdPipeline(pipeline, fwdPD, inVals_, out); +} + +void MKLDNNAddtoLayer::resetBwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + resetBwdBuffers(inGrads_, out); + in = inGrads_[0]; + + // backward only need share output grad to input grad + for (size_t i = 0; i < inGrads_.size(); i++) { + if (inGrads_[i] != nullptr) { + inGrads_[i] = out; + inputLayers_[i]->getOutputGrad()->setData(inGrads_[i]->getData()); + } + } +} + +void MKLDNNAddtoLayer::updateWeights(const UpdateCallback& callback) { + if (biases_ && biases_->getWGrad()) { + biases_->getParameterPtr()->incUpdate(callback); + } +} + +void MKLDNNAddtoLayer::resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& out) { + inputs.resize(inputLayers_.size()); + for (size_t i = 0; i < inputs.size(); i++) { + resetInValue(inputs[i], nullptr, i); + CHECK(inputs[i]); + inputs[i]->downSpatial(); + } + for (size_t i = 1; i < inputs.size(); i++) { + CHECK_PRIMITIVE_DESC_EQ(inputs[i], inputs[0]->getPrimitiveDesc()); + } + + resetOutValue(out, inputs[0]->getPrimitiveDesc()); +} + +void MKLDNNAddtoLayer::resetFwdPD(std::shared_ptr& pd, + std::vector& inputs, + MKLDNNMatrixPtr out) { + std::vector scales(inputs.size(), 1.0); + std::vector srcPDs; + for (size_t i = 0; i < inputs.size(); i++) { + srcPDs.push_back(inputs[i]->getPrimitiveDesc()); + } + CHECK(out); + pd.reset(new sum::primitive_desc(out->getMemoryDesc(), scales, srcPDs)); + CHECK_PRIMITIVE_DESC_EQ(out, pd->dst_primitive_desc()); +} + +void MKLDNNAddtoLayer::resetFwdPipeline( + std::vector& pipeline, + std::shared_ptr& pd, + std::vector& inputs, + MKLDNNMatrixPtr& out) { + std::vector srcs; + for (size_t i = 0; i < inputs.size(); i++) { + srcs.push_back(*(inputs[i])); + } + fwd_.reset(new sum(*pd, srcs, *out)); + pipeline.push_back(*fwd_); +} + +void MKLDNNAddtoLayer::resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& out) { + CHECK(outVal_); + resetOutGrad(out, outVal_->getPrimitiveDesc()); + CHECK(out); + + inputs.resize(inputLayers_.size()); + for (size_t i = 0; i < inputs.size(); i++) { + resetInGrad(inputs[i], inVal_->getPrimitiveDesc(), i); + CHECK_PRIMITIVE_DESC_EQ(inputs[i], out->getPrimitiveDesc()); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.h b/paddle/gserver/layers/MKLDNNAddtoLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..15f74ec5bdf3d1e4ae5e09051be6be418590a67a --- /dev/null +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.h @@ -0,0 +1,110 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "MKLDNNLayer.h" +#include "mkldnn.hpp" + +namespace paddle { + +/** + * @brief A subclass of MKLDNNLayer Addto layer. + * + * The config file api is mkldnn_addto + */ +class MKLDNNAddtoLayer : public MKLDNNLayer { +protected: + std::vector inVals_; + std::vector inGrads_; + + // layer size == ic * ih * iw == oc * oh *ow, and can not be changed + size_t layerSize_; + + // TODO(TJ): this part has not been optimized by MKL-DNN + std::unique_ptr biases_; + +public: + explicit MKLDNNAddtoLayer(const LayerConfig& config) : MKLDNNLayer(config) {} + + ~MKLDNNAddtoLayer() {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void reshape( + int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) override; + + void resetFwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) override; + + void resetBwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) override; + + void updateWeights(const UpdateCallback& callback) override; + + void printValueFormat() override { + for (size_t i = 0; i < inVals_.size(); ++i) { + VLOG(MKLDNN_FMTS) << i << " input: " << inVals_[i]->getFormat() << " >>>"; + } + if (outVal_) { + VLOG(MKLDNN_FMTS) << outVal_->getFormat() << " >>> "; + } + if (extOutVal_) { + VLOG(MKLDNN_FMTS) << extOutVal_->getFormat(); + } + } + + void printGradFormat() override { + if (extOutGrad_) { + VLOG(MKLDNN_FMTS) << extOutGrad_->getFormat(); + } + if (outGrad_) { + VLOG(MKLDNN_FMTS) << outGrad_->getFormat() << " <<< "; + } + for (size_t i = 0; i < inGrads_.size(); ++i) { + VLOG(MKLDNN_FMTS) << i << " input: " << inGrads_[i]->getFormat() << "<<<"; + } + } + +protected: + /** + * Forward functions: reset buffers(inputs, output, bias), + * reset primitive descriptor, + * reset pipeline. + */ + void resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& out); + void resetFwdPD(std::shared_ptr& pd, + std::vector& inputs, + MKLDNNMatrixPtr out); + void resetFwdPipeline(std::vector& pipeline, + std::shared_ptr& pd, + std::vector& inputs, + MKLDNNMatrixPtr& out); + + /** + * Backward functions: reset buffers(inputs, output, bias) + */ + void resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& out); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNLayer.cpp b/paddle/gserver/layers/MKLDNNLayer.cpp index 663a10509857ec9fb487c1cda1621bdfac1250ac..5fd62f4f73b18df683ccf74143e45054c3631c22 100644 --- a/paddle/gserver/layers/MKLDNNLayer.cpp +++ b/paddle/gserver/layers/MKLDNNLayer.cpp @@ -77,7 +77,7 @@ void MKLDNNLayer::forward(PassType passType) { needResetBwd_ = true; } - if (inputLayers_[0]->getType() == "data") { + if (inputLayers_[0]->getType() == "data" && inputLayers_.size() == 1) { // Update input value data when input layer is "data" type, // since the input value data address might be changed. CHECK(extInVal_); @@ -171,14 +171,16 @@ void MKLDNNLayer::resetWithMatrix(MKLDNNMatrixPtr& dnn, } void MKLDNNLayer::resetInValue( - MKLDNNMatrixPtr& in, const std::shared_ptr& intPD) { + MKLDNNMatrixPtr& in, + const std::shared_ptr& intPD, + size_t inputIdx) { cvtInVal_ = nullptr; extInVal_ = nullptr; in = nullptr; CHECK_GT(bs_ * ic_ * ih_ * iw_, 0); auto extPD = MKLDNNMatrix::createPrimitiveDesc( {bs_, ic_, ih_, iw_}, format::nchw, engine_); - const MatrixPtr& inMat = inputLayers_[0]->getOutputValue(); + const MatrixPtr& inMat = inputLayers_[inputIdx]->getOutputValue(); in = std::dynamic_pointer_cast(inMat); CHECK_EQ(inputIsOnlyMKLDNN(), in != nullptr); if (in == nullptr || in->getFormat() == format::nc) { @@ -216,11 +218,12 @@ void MKLDNNLayer::resetOutValue(MKLDNNMatrixPtr& out, } void MKLDNNLayer::resetInGrad(MKLDNNMatrixPtr& in, - memory::primitive_desc intPD) { + memory::primitive_desc intPD, + size_t inputIdx) { cvtInGrad_ = nullptr; extInGrad_ = nullptr; in = nullptr; - LayerPtr& input = inputLayers_[0]; + LayerPtr& input = inputLayers_[inputIdx]; if (input->getOutputGrad() == nullptr) { // no need input grad return; @@ -245,7 +248,6 @@ void MKLDNNLayer::resetInGrad(MKLDNNMatrixPtr& in, return; } // need create reorder - // TODO(TJ): add macro definition to simplify it CHECK(extInVal_ != nullptr && isPaddleFormat(extInVal_->getFormat())) << "should have external input value and the format must be nchw(nc)"; extInGrad_ = MKLDNNMatrix::create(extInVal_->getPrimitiveDesc(), inMat); diff --git a/paddle/gserver/layers/MKLDNNLayer.h b/paddle/gserver/layers/MKLDNNLayer.h index 2c21a5b2aaecb17a52a5de9a98664068f2255d83..7479c34c92b5231b2521493bc631474d4efd4224 100644 --- a/paddle/gserver/layers/MKLDNNLayer.h +++ b/paddle/gserver/layers/MKLDNNLayer.h @@ -199,7 +199,8 @@ protected: */ void resetInValue( MKLDNNMatrixPtr& in, - const std::shared_ptr& intPD = nullptr); + const std::shared_ptr& intPD = nullptr, + size_t inputIdx = 0); /** * reset output value from internal primitive desc. @@ -212,7 +213,9 @@ protected: * reset input grad from internal primitive desc. * reset both internal and external buffer and create reorder if necessary. */ - void resetInGrad(MKLDNNMatrixPtr& in, mkldnn::memory::primitive_desc intPD); + void resetInGrad(MKLDNNMatrixPtr& in, + mkldnn::memory::primitive_desc intPD, + size_t inputIdx = 0); /** * reset output grad from internal primitive desc. diff --git a/paddle/gserver/layers/SequenceReshapeLayer.cpp b/paddle/gserver/layers/SequenceReshapeLayer.cpp index 433592953b220eda4db4634124a57a2074cef4c0..822974407283c9ee6d0efee71bc945bc418b1942 100644 --- a/paddle/gserver/layers/SequenceReshapeLayer.cpp +++ b/paddle/gserver/layers/SequenceReshapeLayer.cpp @@ -70,11 +70,23 @@ void SequenceReshapeLayer::forward(PassType passType) { size_t outDim = getSize(); size_t numSequences = input.getNumSequences(); - auto startPositions = input.sequenceStartPositions->getVector(false); - const int* starts = startPositions->getData(); - CHECK_EQ(starts[numSequences], input.getBatchSize()); - CHECK_EQ(numSequences, startPositions->getSize() - 1); + // by default, we assume each instance as a sequence + IVectorPtr seqStarts; + IVector::resizeOrCreate(seqStarts, input.getBatchSize() + 1, false); + int* startsData = seqStarts->getData(); + for (int i = 0; i < input.getBatchSize() + 1; i++) { + startsData[i] = i; + } + const int* starts = startsData; + + // if there is sequence, then use start positions + if (input.sequenceStartPositions) { + auto startPositions = input.sequenceStartPositions->getVector(false); + starts = startPositions->getData(); + CHECK_EQ(starts[numSequences], input.getBatchSize()); + CHECK_EQ(numSequences, startPositions->getSize() - 1); + } for (size_t seqID = 0; seqID < numSequences; seqID++) { size_t inNumIns = starts[seqID + 1] - starts[seqID]; diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 329536afaf6d69676e8c39fdf8b6b8cb87ade5fa..aa94ee406e27c86e6d49b6d2b5327a3f86bcacd6 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -1,24 +1,29 @@ # gserver pacakge unittests -if(NOT MOBILE_INFERENCE) -################### test_ProtoDataProvider ############ - add_unittest_without_exec(test_ProtoDataProvider - test_ProtoDataProvider.cpp) - - # test_ProtoDataProvider will mkdir as same name, - # so if WORKING_DIRECTORY is default directory, then - # mkdir will get error. - add_test(NAME test_ProtoDataProvider - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ProtoDataProvider - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) -endif() +add_simple_unittest(test_LinearChainCRF) +add_simple_unittest(test_MultinomialSampler) +add_simple_unittest(test_RecurrentLayer) -################# test_LayerGrad ####################### -add_unittest_without_exec(test_LayerGrad - test_LayerGrad.cpp - LayerGradUtil.cpp) -add_test(NAME test_LayerGrad - COMMAND test_LayerGrad) +function(gserver_test TARGET) + add_unittest_without_exec(${TARGET} + ${TARGET}.cpp + LayerGradUtil.cpp) + add_test(NAME ${TARGET} + COMMAND ${TARGET}) +endfunction() + +gserver_test(test_LayerGrad) +gserver_test(test_CRFLayerGrad) +gserver_test(test_CrossEntropyOverBeamGrad) +gserver_test(test_SeqSliceLayerGrad) +gserver_test(test_ActivationGrad) +gserver_test(test_ConvTrans) +gserver_test(test_PriorBox) +gserver_test(test_DetectionOutput) +gserver_test(test_ConvUnify) +gserver_test(test_BatchNorm) +gserver_test(test_KmaxSeqScore) +gserver_test(test_Expand) ########## test_Mkldnn layers and activations ########## if(WITH_MKLDNN) @@ -32,89 +37,6 @@ if(WITH_MKLDNN) WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) endif() -################ test_CRFLayerGrad #################### -add_unittest_without_exec(test_CRFLayerGrad - test_CRFLayerGrad.cpp - LayerGradUtil.cpp) -add_test(NAME test_CRFLayerGrad - COMMAND test_CRFLayerGrad) - -################ test_CrossEntropyOverBeam #################### -add_unittest_without_exec(test_CrossEntropyOverBeam - test_CrossEntropyOverBeamGrad.cpp - LayerGradUtil.cpp) -add_test(NAME test_CrossEntropyOverBeam - COMMAND test_CrossEntropyOverBeam) - -################ test_SeqSliceLayerGrad #################### -add_unittest_without_exec(test_SeqSliceLayerGrad - test_SeqSliceLayerGrad.cpp - LayerGradUtil.cpp) -add_test(NAME test_SeqSliceLayerGrad - COMMAND test_SeqSliceLayerGrad) - -add_unittest_without_exec(test_ActivationGrad - test_ActivationGrad.cpp - LayerGradUtil.cpp) -add_test(NAME test_ActivationGrad - COMMAND test_ActivationGrad) -################# test_ConvTrans ####################### -add_unittest_without_exec(test_ConvTrans - test_ConvTrans.cpp - LayerGradUtil.cpp) - -add_test(NAME test_ConvTrans - COMMAND test_ConvTrans) -################# test_PriorBox ####################### -add_unittest_without_exec(test_PriorBox - test_PriorBox.cpp - LayerGradUtil.cpp) - -add_test(NAME test_PriorBox - COMMAND test_PriorBox) -################# test_DetectionOutput ####################### -add_unittest_without_exec(test_DetectionOutput - test_DetectionOutput.cpp - LayerGradUtil.cpp) - -add_test(NAME test_DetectionOutput - COMMAND test_DetectionOutput) -################# test_ConvUnify ####################### -add_unittest_without_exec(test_ConvUnify - test_ConvUnify.cpp - LayerGradUtil.cpp) - -add_test(NAME test_ConvUnify - COMMAND test_ConvUnify) -################# test_BatchNorm ####################### -add_unittest_without_exec(test_BatchNorm - test_BatchNorm.cpp - LayerGradUtil.cpp) - -add_test(NAME test_BatchNorm - COMMAND test_BatchNorm) - - -################# test_KmaxSeqScore ####################### -add_unittest_without_exec(test_KmaxSeqScore - test_KmaxSeqScore.cpp - LayerGradUtil.cpp) - -add_test(NAME test_KmaxSeqScore - COMMAND test_KmaxSeqScore) - -if(NOT MOBILE_INFERENCE) -################## test_Evaluator ####################### - add_unittest(test_Evaluator - test_Evaluator.cpp) -endif() - -################ test_LinearChainCRF #################### -add_simple_unittest(test_LinearChainCRF) - -############## test_MultinomialSampler ################### -add_simple_unittest(test_MultinomialSampler) - ############## test_PyDataProvider ######################## if(WITH_PYTHON) add_unittest_without_exec(test_PyDataProvider @@ -125,9 +47,6 @@ if(WITH_PYTHON) WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) endif() -############### test_RecurrentLayer ####################### -add_simple_unittest(test_RecurrentLayer) - ############### test_WarpCTCLayer ####################### if(NOT WITH_DOUBLE) add_unittest_without_exec(test_WarpCTCLayer @@ -139,19 +58,33 @@ if(NOT WITH_DOUBLE) endif() if(NOT MOBILE_INFERENCE) -############### test_RecurrentGradientMachine ############### - # TODO(yuyang18): There is some bug in test_RecurrentGradientMachine - # I will fix it. - add_unittest_without_exec(test_RecurrentGradientMachine - test_RecurrentGradientMachine.cpp) - add_test(NAME test_RecurrentGradientMachine - COMMAND .set_python_path.sh -d - ${PADDLE_SOURCE_DIR}/python:${PADDLE_SOURCE_DIR}/paddle/gserver/tests - ${CMAKE_CURRENT_BINARY_DIR}/test_RecurrentGradientMachine - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) -endif() +################### test_ProtoDataProvider ############ + add_unittest_without_exec(test_ProtoDataProvider + test_ProtoDataProvider.cpp) -if(NOT MOBILE_INFERENCE) + # test_ProtoDataProvider will mkdir as same name, + # so if WORKING_DIRECTORY is default directory, then + # mkdir will get error. + add_test(NAME test_ProtoDataProvider + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ProtoDataProvider + WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) + +################## test_Evaluator ####################### + add_unittest(test_Evaluator + test_Evaluator.cpp) + +############### test_RecurrentGradientMachine ############### + # TODO(yuyang18): There is some bug in test_RecurrentGradientMachine + # I will fix it. + add_unittest_without_exec(test_RecurrentGradientMachine + test_RecurrentGradientMachine.cpp) + add_test(NAME test_RecurrentGradientMachine + COMMAND .set_python_path.sh -d + ${PADDLE_SOURCE_DIR}/python:${PADDLE_SOURCE_DIR}/paddle/gserver/tests + ${CMAKE_CURRENT_BINARY_DIR}/test_RecurrentGradientMachine + WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle) + +############### test_NetworkCompare ############### add_unittest_without_exec(test_NetworkCompare test_NetworkCompare.cpp) if(WITH_GPU) diff --git a/paddle/gserver/tests/MKLDNNTester.cpp b/paddle/gserver/tests/MKLDNNTester.cpp index 73b7e8857f35d194e71b2b5b341f89b77fd1f8b0..afe1608eab8eaf1217a7a0c8a2774e37c5ea83f4 100644 --- a/paddle/gserver/tests/MKLDNNTester.cpp +++ b/paddle/gserver/tests/MKLDNNTester.cpp @@ -132,7 +132,7 @@ void MKLDNNTester::checkForward() { VLOG(MKLDNN_TESTS) << "Check Forward"; printTopDatas(); double delta = - compareMatrix(dnnLayer_->getOutputValue(), refLayer_->getOutputValue()); + compareMatrix(refLayer_->getOutputValue(), dnnLayer_->getOutputValue()); EXPECT_LE(fabs(delta), eps_); } @@ -147,7 +147,7 @@ void MKLDNNTester::checkBackwardData() { VLOG(MKLDNN_ALL) << "Reference Backward Result: InputGrad " << i; printMatrix(refDiff); - double delta = compareMatrix(dnnDiff, refDiff); + double delta = compareMatrix(refDiff, dnnDiff); EXPECT_LE(fabs(delta), eps_); if (isBN) { // the other two inputs in batch norm are for moving mean and var @@ -177,7 +177,7 @@ void MKLDNNTester::checkBackwardWgts() { << parameters_[REF][i]->getName(); printVector(ref); - double delta = compareVector(dnn, ref); + double delta = compareVector(ref, dnn); EXPECT_LE(fabs(delta), eps_); } @@ -273,31 +273,37 @@ void MKLDNNTester::printVector(const VectorPtr& v) { VLOG(MKLDNN_ALL) << std::endl << ostr.str(); } -double MKLDNNTester::getDelta(const real* d1, - const real* d2, +double MKLDNNTester::getDelta(const real* refer, + const real* value, size_t len, const float failRate, const float thres) { double delta = 0, sum = 0; int failCnt = 0; const double eps = 1e-5; - double maxOut = 0; + double maxRatio = 0; for (size_t i = 0; i < len; ++i) { - double ref = fabs(d2[i]); - double diff = fabs(d1[i] - d2[i]); + double ref = fabs(refer[i]); + double val = fabs(value[i]); + double diff = fabs(refer[i] - value[i]); delta += diff; sum += ref; - if (ref > eps && fabs(d1[i]) > eps && diff / ref > thres) { - maxOut = std::max(maxOut, diff / ref); + if (ref < eps && val < eps) { // both values are very small + continue; + } + double ratio = diff / ref; + if (ratio > thres) { + maxRatio = std::max(maxRatio, ratio); failCnt++; } } - EXPECT_TRUE(std::isnormal(sum)); EXPECT_FALSE(std::isinf(sum)); + EXPECT_FALSE(std::isnan(sum)); EXPECT_FALSE(std::isnan(delta)); VLOG(MKLDNN_ALL) << "reference avg data: " << sum / len << ", delta: " << delta / sum << ", failCnt:" << failCnt; - return (failCnt / (float)len) > failRate ? maxOut : delta / sum; + double res = sum > eps ? delta / sum : eps; + return (failCnt / (float)len) > failRate ? maxRatio : res; } double MKLDNNTester::compareMatrix(const MatrixPtr& m1, const MatrixPtr& m2) { @@ -515,12 +521,16 @@ void MKLDNNTester::getOutResult(const std::string& configPath, gradientMachine->forward(in.inArgs[i], &outArgs, PASS_TRAIN); // save forward result for (size_t k = 0; k < outArgs.size(); k++) { - MatrixPtr value = Matrix::create(outArgs[k].value->getHeight(), - outArgs[k].value->getWidth(), - false, - false); - value->copyFrom(*outArgs[k].value); - out.outValues.push_back(value); + const MatrixPtr& src = outArgs[k].value; + MatrixPtr dst = + Matrix::create(src->getHeight(), src->getWidth(), false, false); + if (typeid(*src) == typeid(MKLDNNMatrix)) { + MKLDNNMatrixPtr dnnSrc = std::dynamic_pointer_cast(src); + dnnSrc->copyTo(*dst); + } else { + dst->copyFrom(*src); + } + out.outValues.push_back(dst); } // random backward input @@ -543,19 +553,19 @@ void MKLDNNTester::getOutResult(const std::string& configPath, void MKLDNNTester::compareResult(DataOut& ref, DataOut& dnn, float eps) { CHECK_EQ(ref.outValues.size(), dnn.outValues.size()); CHECK_EQ(ref.paraValues.size(), dnn.paraValues.size()); - VLOG(MKLDNN_TESTS) << "compare value size: " << ref.outValues.size(); for (size_t i = 0; i < ref.outValues.size(); i++) { + VLOG(MKLDNN_TESTS) << "compare value index: " << i; EXPECT_LE(fabs(compareMatrix(ref.outValues[i], dnn.outValues[i])), eps); } - VLOG(MKLDNN_TESTS) << "compare param size: " << ref.outValues.size(); for (size_t i = 0; i < ref.paraValues.size(); i++) { + VLOG(MKLDNN_TESTS) << "compare param index: " << i; EXPECT_LE(fabs(compareVector(ref.paraValues[i], dnn.paraValues[i])), eps); } } -void MKLDNNTester::runBranchesTest(const std::string& configPath, - size_t iter, - float eps) { +void MKLDNNTester::runNetTest(const std::string& configPath, + size_t iter, + float eps) { DataIn in; initArgument(in, configPath, iter); DataOut outCpu, outDnn; diff --git a/paddle/gserver/tests/MKLDNNTester.h b/paddle/gserver/tests/MKLDNNTester.h index 19d8848f74f2ee4a809e42164a0eb180abd2a4e1..ca55a45bc77b4e171619ab788d7c7dfeefcd036a 100644 --- a/paddle/gserver/tests/MKLDNNTester.h +++ b/paddle/gserver/tests/MKLDNNTester.h @@ -85,17 +85,17 @@ public: bool printDetails = false, size_t iter = 3, float epsilon = 1e-4); - static void runBranchesTest(const std::string& configPath, - size_t iter = 3, - float eps = 1e-4); + static void runNetTest(const std::string& configPath, + size_t iter = 2, + float eps = 1e-4); static void initArgument(DataIn& data, const std::string& configPath, - size_t iter = 3); + size_t iter = 2); static void getOutResult(const std::string& configPath, DataIn& in, DataOut& out, bool use_mkldnn, - size_t iter = 3); + size_t iter = 2); private: void reset(const TestConfig& dnn, const TestConfig& ref, size_t batchSize); @@ -128,13 +128,13 @@ private: /** * Get delta percent - * if many(>failRate) wrong(abs(dnn-ref)/abs(ref)>thres) points return the - * max(diff/ref) - * else return sum(abs(a-b)) / sum(abs(b)) + * if many(>failRate) wrong(abs(val-ref)/abs(ref) > thres) points + * return the max(diff/ref) + * else return sum(abs(diff)) / sum(abs(ref)) * The return value should be smaller than eps when passing. */ - static double getDelta(const real* d1, - const real* d2, + static double getDelta(const real* refer, + const real* value, size_t len, const float failRate = 1e-3, const float thres = 0.1); diff --git a/paddle/gserver/tests/mkldnn_branch_net.conf b/paddle/gserver/tests/mkldnn_branch_net.conf new file mode 100644 index 0000000000000000000000000000000000000000..8d5146abb0ebd7f5d6c512457f3cb5c84eac20f5 --- /dev/null +++ b/paddle/gserver/tests/mkldnn_branch_net.conf @@ -0,0 +1,142 @@ +# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=16) +channels = get_config_arg("channels", int, 2) + +def two_conv(input, group_name): + out1 = img_conv_layer(input=input, + name=group_name+'_conv1_', + filter_size=1, + num_filters=channels, + padding=0, + shared_biases=True, + act=ReluActivation()) + + out2 = img_conv_layer(input=input, + name=group_name+'_conv2_', + filter_size=3, + num_filters=channels, + padding=1, + shared_biases=True, + act=ReluActivation()) + return out1, out2 + +def two_conv_bn(input, group_name): + out1, out2 = two_conv(input, group_name) + out1 = batch_norm_layer(input=out1, + name=group_name+'_bn1_', + use_global_stats=False, + act=ReluActivation()) + + out2 = batch_norm_layer(input=out2, + name=group_name+'_bn2_', + use_global_stats=False, + act=ReluActivation()) + return out1, out2 + +def two_conv_pool(input, group_name): + out1, out2 = two_conv(input, group_name) + out1 = img_pool_layer(input=out1, + name=group_name+'_pool1_', + pool_size=3, + stride=2, + padding=0, + pool_type=MaxPooling()) + + out2 = img_pool_layer(input=out2, + name=group_name+'_pool2_', + pool_size=5, + stride=2, + padding=1, + pool_type=MaxPooling()) + return out1, out2 + +def two_fc(input, group_name): + out1 = fc_layer(input=input, + name=group_name+'_fc1_', + size=channels, + bias_attr=False, + act=LinearActivation()) + + out2 = fc_layer(input=input, + name=group_name+'_fc2_', + size=channels, + bias_attr=False, + act=LinearActivation()) + return out1, out2 + +data = data_layer(name ="input", size=channels*16*16) + +tmp = img_conv_layer(input=data, + num_channels=channels, + filter_size=3, + num_filters=channels, + padding=1, + shared_biases=True, + act=ReluActivation()) + +a1, a2 = two_conv(tmp, 'conv_branch') +tmp = addto_layer(input=[a1, a2], + act=ReluActivation(), + bias_attr=False) + +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=2, + padding=1, + pool_type=AvgPooling()) + +b1, b2 = two_conv_pool(tmp, 'pool_branch') +tmp = concat_layer(input=[b1, b2]) + +tmp = img_pool_layer(input=tmp, + num_channels=channels*2, + pool_size=3, + stride=2, + padding=1, + pool_type=MaxPooling()) + +tmp = img_conv_layer(input=tmp, + filter_size=3, + num_filters=channels, + padding=1, + stride=2, + shared_biases=True, + act=LinearActivation(), + bias_attr=False) + +tmp = batch_norm_layer(input=tmp, + use_global_stats=False, + act=ReluActivation()) + +c1, c2 = two_conv_bn(tmp, 'bn_branch') +tmp = addto_layer(input=[c1, c2], + act=ReluActivation(), + bias_attr=False) + +tmp = fc_layer(input=tmp, size=channels, + bias_attr=True, + act=ReluActivation()) + +d1, d2 = two_fc(tmp, 'fc_branch') +tmp = addto_layer(input=[d1, d2]) + +out = fc_layer(input=tmp, size=10, + bias_attr=True, + act=SoftmaxActivation()) + +outputs(out) diff --git a/paddle/gserver/tests/mkldnn_branches_fc.conf b/paddle/gserver/tests/mkldnn_branches_fc.conf deleted file mode 100644 index fb85425c2b63c7604d636e2b0c5d20d91fb5de1b..0000000000000000000000000000000000000000 --- a/paddle/gserver/tests/mkldnn_branches_fc.conf +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -settings(batch_size=16) -channels = get_config_arg("channels", int, 2) - -def two_fc(input, group_name): - out1 = fc_layer(input=input, - name=group_name+'_fc1', - size=channels, - bias_attr=False, - act=LinearActivation()) - - out2 = fc_layer(input=input, - name=group_name+'_fc2', - size=channels, - bias_attr=False, - act=LinearActivation()) - return out1, out2 - -data = data_layer(name ="input", size=channels*16*16) - -conv = img_conv_layer(input=data, - num_channels=channels, - filter_size=3, - num_filters=channels, - padding=1, - shared_biases=True, - act=LinearActivation()) - -pool = img_pool_layer(input=conv, - pool_size=3, - stride=2, - padding=1, - pool_type=AvgPooling()) - -a1, a2 = two_fc(input=pool, group_name='a') - -concat = concat_layer(input=[a1, a2]) - -b1, b2 = two_fc(input=pool, group_name='b') - -addto = addto_layer(input=[b1, b2]) - -outputs([concat, addto]) diff --git a/paddle/gserver/tests/mkldnn_branches_pool.conf b/paddle/gserver/tests/mkldnn_branches_pool.conf deleted file mode 100644 index ca17c74752ab0777a69f818d9f43275a6140cb4c..0000000000000000000000000000000000000000 --- a/paddle/gserver/tests/mkldnn_branches_pool.conf +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -settings(batch_size=16) -channels = get_config_arg("channels", int, 2) - -def two_pool(input, group_name): - out1 = img_pool_layer(input=input, - name=group_name+'_pool1', - pool_size=3, - stride=2, - padding=0, - pool_type=MaxPooling()) - - out2 = img_pool_layer(input=input, - name=group_name+'_pool2', - pool_size=5, - stride=2, - padding=1, - pool_type=MaxPooling()) - return out1, out2 - -data = data_layer(name ="input", size=channels*16*16) - -conv = img_conv_layer(input=data, - num_channels=channels, - filter_size=3, - num_filters=channels, - padding=1, - shared_biases=True, - act=LinearActivation()) - -pool = img_pool_layer(input=conv, - pool_size=3, - stride=1, - padding=1, - pool_type=AvgPooling()) - -a1, a2 = two_pool(input=pool, group_name='a') - -concat = concat_layer(input=[a1, a2]) - -b1, b2 = two_pool(input=pool, group_name='b') - -addto = addto_layer(input=[b1, b2]) - -outputs([concat, addto]) diff --git a/paddle/gserver/tests/mkldnn_branches_conv.conf b/paddle/gserver/tests/mkldnn_simple_net.conf similarity index 64% rename from paddle/gserver/tests/mkldnn_branches_conv.conf rename to paddle/gserver/tests/mkldnn_simple_net.conf index 2628509db43e6a5f69a4f5ea956bffdc2837e32a..8bbe91e56d0ba6da06475ad16f3162ee1103ee02 100644 --- a/paddle/gserver/tests/mkldnn_branches_conv.conf +++ b/paddle/gserver/tests/mkldnn_simple_net.conf @@ -17,40 +17,48 @@ from paddle.trainer_config_helpers import * settings(batch_size=16) channels = get_config_arg("channels", int, 2) -def two_conv(input, group_name): - out1 = img_conv_layer(input=input, - name=group_name+'_conv1', - filter_size=1, - num_filters=channels, - padding=0, - shared_biases=True, - act=ReluActivation()) +data = data_layer(name ="input", size=channels*16*16) - out2 = img_conv_layer(input=input, - name=group_name+'_conv2', +tmp = img_conv_layer(input=data, + num_channels=channels, filter_size=3, num_filters=channels, padding=1, shared_biases=True, act=ReluActivation()) - return out1, out2 -data = data_layer(name ="input", size=channels*16*16) +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=1, + padding=0, + pool_type=AvgPooling()) -conv = img_conv_layer(input=data, - num_channels=channels, +tmp = img_conv_layer(input=tmp, filter_size=3, num_filters=channels, padding=1, shared_biases=True, - act=ReluActivation()) + act=LinearActivation(), + bias_attr=False) -a1, a2 = two_conv(input=conv, group_name='a') +tmp = batch_norm_layer(input=tmp, + use_global_stats=False, + act=ReluActivation()) -concat = concat_layer(input=[a1, a2]) +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=2, + padding=1, + pool_type=MaxPooling()) -b1, b2 = two_conv(input=conv, group_name='b') +tmp = fc_layer(input=tmp, + size=channels, + bias_attr=False, + act=ReluActivation()) -addto = addto_layer(input=[b1, b2]) +out = fc_layer(input=tmp, + size=10, + bias_attr=True, + act=SoftmaxActivation()) -outputs([concat, addto]) +outputs(out) diff --git a/paddle/gserver/tests/test_Expand.cpp b/paddle/gserver/tests/test_Expand.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d32bf0152f77bba098daa508fe448784ac013549 --- /dev/null +++ b/paddle/gserver/tests/test_Expand.cpp @@ -0,0 +1,127 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include + +#include "LayerGradUtil.h" +#include "paddle/testing/TestUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +// Do one forward pass of expand layer and check to see if its output +// matches the given result.(Test onlyCPU currently.) +void doOneExpandTest(string trans_type, + bool hasSubseq, + bool useGpu, + Argument& input1, + Argument& input2, + Argument& result) { + FLAGS_use_gpu = false; + // Setting up the expand layer + TestConfig config; + config.layerConfig.set_type("expand"); + + auto inputType1 = + trans_type == "non-seq" ? INPUT_DENSE_DIM_DATA : INPUT_SEQUENCE_DATA; + config.inputDefs.push_back({inputType1, "layer0", 1, 0}); + auto inputType2 = + hasSubseq ? INPUT_HASSUB_SEQUENCE_DATA : INPUT_SEQUENCE_DATA; + + config.inputDefs.push_back({inputType2, "layer1", 1, 0}); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + config.layerConfig.set_trans_type(trans_type); + + // data layer initialize + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer( + config, &dataLayers, &datas, &layerMap, "expand", 1, false, useGpu); + dataLayers[0]->getOutput() = input1; + dataLayers[1]->getOutput() = input2; + + // test layer initialize + std::vector parameters; + LayerPtr expandLayer; + initTestLayer(config, &layerMap, ¶meters, &expandLayer); + expandLayer->forward(PASS_GC); + checkMatrixEqual(expandLayer->getOutputValue(), result.value); +} + +TEST(Layer, ExpandLayerFwd) { + bool useGpu = false; + + // Assume batch_size =3 in all cases. + + // CPU case 1. non-seq expand to seq + // input1 = 1,2,3 + // input2 = [4,5],[6],[7,8,9] + // result = [1,1],[2],[3,3,3] + Argument input1, input2, result; + input1.value = Matrix::create(3, 1, false, useGpu); + real input1Data[] = {1, 2, 3}; + input1.value->setData(input1Data); + + input2.value = Matrix::create(6, 1, false, useGpu); + real input2Data[] = {4, 5, 6, 7, 8, 9}; + input2.value->setData(input2Data); + input2.sequenceStartPositions = ICpuGpuVector::create(4, useGpu); + int input2Seq[] = {0, 2, 3, 6}; + input2.sequenceStartPositions->copyFrom(input2Seq, 4, useGpu); + + result.value = Matrix::create(6, 1, false, useGpu); + real resultData[] = {1, 1, 2, 3, 3, 3}; + result.value->setData(resultData); + + doOneExpandTest("non-seq", false, useGpu, input1, input2, result); + + // CPU case 2. non-seq expand to sub-seq + // NOTE: input1.batch_size == input2.sequencelength in this case. + // i.e, input1 expands by input2.sequence + // input1 = 1,2,3 + // input2 = [[4,5]],[[6]],[[7],[8,9]] + // result = [[1,1]],[[2]],[[3],[3,3]] + input2.subSequenceStartPositions = ICpuGpuVector::create(5, useGpu); + int input2SubSeq[] = {0, 2, 3, 4, 6}; + input2.subSequenceStartPositions->copyFrom(input2SubSeq, 5, useGpu); + + doOneExpandTest("non-seq", true, useGpu, input1, input2, result); + + // CPU case 3. seq expand to sub-seq + // input1 = [1,2],[3],[4] + // input2 = [[4,5]],[[6]],[[7],[8,9]] + // result = [[1,1]],[[2]],[[3],[4,4]] + Matrix::resizeOrCreate(input1.value, 4, 1, false, useGpu); + real input1Data_case3[] = {1, 2, 3, 4}; + input1.value->setData(input1Data_case3); + + input1.sequenceStartPositions = ICpuGpuVector::create(4, useGpu); + int input1Seq[] = {0, 2, 3, 4}; + input1.sequenceStartPositions->copyFrom(input1Seq, 4, useGpu); + + real resultData_case3[] = {1, 1, 2, 3, 4, 4}; + result.value->setData(resultData_case3); + + doOneExpandTest("seq", true, useGpu, input1, input2, result); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 85d4f437c2664135a7975c6ed3270d8f1ddbeaf4..2e8d9f3333b36005c9b3b28449c76a4a44c74cc6 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -234,8 +234,7 @@ static void getMKLDNNBatchNormConfig(TestConfig& cfg, cfg.inputDefs.push_back({INPUT_DATA, "layer_2_moving_var", 1, size_t(pm.ic)}); cfg.inputDefs.back().isStatic = true; LayerInputConfig* input = cfg.layerConfig.add_inputs(); - // TODO(TJ): uncomment me when refine and support comparing all zeroes vector - // cfg.layerConfig.set_active_type("relu"); + cfg.layerConfig.set_active_type("relu"); cfg.layerConfig.add_inputs(); cfg.layerConfig.add_inputs(); ImageConfig* img_conf = input->mutable_image_conf(); @@ -272,20 +271,53 @@ TEST(MKLDNNLayer, BatchNormLayer) { testBatchNormLayer({16, 32, 16, 16}); } -struct testActDesc { +struct testImageDesc { int bs, ic, ih, iw; }; -static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { +static void getAddtoConfig(TestConfig& cfg, + const testImageDesc& pm, + const size_t nInputs = 1) { cfg.biasSize = 0; cfg.layerConfig.set_type("addto"); size_t layerSize = pm.ic * pm.ih * pm.iw; cfg.layerConfig.set_size(layerSize); - cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); - cfg.layerConfig.add_inputs(); + cfg.layerConfig.set_active_type("relu"); + for (size_t i = 0; i < nInputs; ++i) { + std::stringstream ss; + ss << "layer_" << i; + cfg.inputDefs.push_back({INPUT_DATA, ss.str(), layerSize, 0}); + LayerInputConfig* input = cfg.layerConfig.add_inputs(); + ImageConfig* img_conf = input->mutable_image_conf(); + img_conf->set_channels(pm.ic); + img_conf->set_img_size_y(pm.ih); + img_conf->set_img_size(pm.iw); + } +} + +void testAddtoLayer(const testImageDesc& pm, const size_t nInputs) { + CHECK_GE(nInputs, 1); + TestConfig dnnConfig; + getAddtoConfig(dnnConfig, pm, nInputs); + dnnConfig.layerConfig.set_type("mkldnn_addto"); + // TODO(TJ): test with bias + for (auto withBias : {false}) { + if (withBias) { + dnnConfig.biasSize = pm.ic * pm.ih * pm.iw; + } else { + dnnConfig.biasSize = 0; + } + RUN_MKLDNN_TEST_LAYER(dnnConfig, "addto", pm) + } +} + +TEST(MKLDNNLayer, AddtoLayer) { + testAddtoLayer({16, 5, 14, 14}, 1); + testAddtoLayer({8, 10, 8, 8}, 2); + testAddtoLayer({4, 12, 1, 1}, 3); } -void testActivation(std::string actType, const testActDesc& pm) { +void testActivation(std::string actType, const testImageDesc& pm) { // TODO(TJ): remove me when paddle support elu activation if (actType == "mkldnn_elu") { return; @@ -309,15 +341,15 @@ TEST(MKLDNNActivation, Activations) { } DECLARE_string(config_args); -TEST(MKLDNNLayer, branches) { - std::vector cases = {"conv", "pool", "fc"}; +TEST(MKLDNNNet, net) { + std::vector cases = {"simple", "branch"}; for (auto name : cases) { - std::string config = "./gserver/tests/mkldnn_branches_" + name + ".conf"; + std::string config = "./gserver/tests/mkldnn_" + name + "_net.conf"; for (auto channels : {2, 32}) { std::ostringstream oss; oss << "channels=" << channels; FLAGS_config_args = oss.str(); - MKLDNNTester::runBranchesTest(config); + MKLDNNTester::runNetTest(config); } } } diff --git a/paddle/math/MKLDNNMatrix.h b/paddle/math/MKLDNNMatrix.h index 5f5b819017b83579ce58522198b3f13311297d42..54cfefe23b3dc70fd12fd2ca8886c941047b59f7 100644 --- a/paddle/math/MKLDNNMatrix.h +++ b/paddle/math/MKLDNNMatrix.h @@ -102,6 +102,11 @@ public: m_->copyFrom(src); } + void copyTo(Matrix& dst) { + // TODO(TJ): reorder data if this format is not nchw or x + dst.copyFrom(*m_); + } + public: /** * Reorder this MKLDNNMatrix from other format. diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index ff72672e3ab77212b309fcfea835839a916fa632..346008439c35a2bcbcd2e9dfd36d689e01d7495f 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -18,6 +18,7 @@ limitations under the License. */ #include #include "Matrix.h" #include "hl_gpu.h" +#include "hl_matrix.h" #include "hl_table_apply.h" #include "paddle/utils/Flags.h" #include "paddle/utils/Logging.h" @@ -99,6 +100,19 @@ MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { return mat; } +template <> +std::shared_ptr> VectorT::castToInt() { + std::shared_ptr> ret = IVector::create(this->getSize(), useGpu_); + if (useGpu_) { + hl_vector_cast2int(ret->getData(), this->getData(), this->getSize()); + } else { + for (size_t i = 0; i < getSize(); ++i) { + ret->getData()[i] = int(this->getData()[i]); + } + } + return ret; +} + template GpuVectorT::GpuVectorT(size_t size) : VectorT(size, diff --git a/paddle/math/Vector.h b/paddle/math/Vector.h index 80b9775fccf10c57bb48145ef56165ec7c86d8b8..f965a5809209da313c78a545c44e7aa39e95ac65 100644 --- a/paddle/math/Vector.h +++ b/paddle/math/Vector.h @@ -162,6 +162,13 @@ public: */ std::shared_ptr toOneHotSparseMatrix(size_t idRange, bool useGpu); + /** + * @brief cast vector of "real" elements to "int" elements. + * + * @note: float -> int must be casted, or you'll get wrong data. + */ + std::shared_ptr> castToInt(); + /** * This function will crash if the size of src and dest is different. */ diff --git a/paddle/memory/detail/buddy_allocator.cc b/paddle/memory/detail/buddy_allocator.cc index e212f7737a4093125857126cabb5b1a7b3e055b1..64ee53803891f192302bb915027f0499dfa36411 100644 --- a/paddle/memory/detail/buddy_allocator.cc +++ b/paddle/memory/detail/buddy_allocator.cc @@ -27,11 +27,11 @@ BuddyAllocator::BuddyAllocator(SystemAllocator* system_allocator, system_allocator_(std::move(system_allocator)) {} BuddyAllocator::~BuddyAllocator() { - VLOG(3) << "BuddyAllocator Disconstructor makes sure that all of these " - "have actually been freed"; + VLOG(10) << "BuddyAllocator Disconstructor makes sure that all of these " + "have actually been freed"; while (!pool_.empty()) { auto block = static_cast(std::get<2>(*pool_.begin())); - VLOG(3) << "Free from block (" << block << ", " << max_chunk_size_ << ")"; + VLOG(10) << "Free from block (" << block << ", " << max_chunk_size_ << ")"; system_allocator_->Free(block, max_chunk_size_, block->index(cache_)); cache_.invalidate(block); @@ -51,11 +51,12 @@ void* BuddyAllocator::Alloc(size_t unaligned_size) { // acquire the allocator lock std::lock_guard lock(mutex_); - VLOG(3) << "Allocate " << unaligned_size << " bytes from chunk size " << size; + VLOG(10) << "Allocate " << unaligned_size << " bytes from chunk size " + << size; // if the allocation is huge, send directly to the system allocator if (size > max_chunk_size_) { - VLOG(3) << "Allocate from system allocator."; + VLOG(10) << "Allocate from system allocator."; return SystemAlloc(size); } @@ -70,9 +71,9 @@ void* BuddyAllocator::Alloc(size_t unaligned_size) { return nullptr; } } else { - VLOG(3) << "Allocation from existing memory block " << std::get<2>(*it) - << " at address " - << reinterpret_cast(std::get<2>(*it))->data(); + VLOG(10) << "Allocation from existing memory block " << std::get<2>(*it) + << " at address " + << reinterpret_cast(std::get<2>(*it))->data(); } total_used_ += size; @@ -89,10 +90,10 @@ void BuddyAllocator::Free(void* p) { // Acquire the allocator lock std::lock_guard lock(mutex_); - VLOG(3) << "Free from address " << block; + VLOG(10) << "Free from address " << block; if (block->type(cache_) == MemoryBlock::HUGE_CHUNK) { - VLOG(3) << "Free directly from system allocator"; + VLOG(10) << "Free directly from system allocator"; system_allocator_->Free(block, block->total_size(cache_), block->index(cache_)); @@ -109,8 +110,8 @@ void BuddyAllocator::Free(void* p) { // Trying to merge the right buddy if (block->has_right_buddy(cache_)) { - VLOG(3) << "Merging this block " << block << " with its right buddy " - << block->right_buddy(cache_); + VLOG(10) << "Merging this block " << block << " with its right buddy " + << block->right_buddy(cache_); auto right_buddy = block->right_buddy(cache_); @@ -127,8 +128,8 @@ void BuddyAllocator::Free(void* p) { // Trying to merge the left buddy if (block->has_left_buddy(cache_)) { - VLOG(3) << "Merging this block " << block << " with its left buddy " - << block->left_buddy(cache_); + VLOG(10) << "Merging this block " << block << " with its left buddy " + << block->left_buddy(cache_); auto left_buddy = block->left_buddy(cache_); @@ -144,8 +145,8 @@ void BuddyAllocator::Free(void* p) { } // Dumping this block into pool - VLOG(3) << "Inserting free block (" << block << ", " - << block->total_size(cache_) << ")"; + VLOG(10) << "Inserting free block (" << block << ", " + << block->total_size(cache_) << ")"; pool_.insert( IndexSizeAddress(block->index(cache_), block->total_size(cache_), block)); @@ -164,7 +165,7 @@ void* BuddyAllocator::SystemAlloc(size_t size) { size_t index = 0; void* p = system_allocator_->Alloc(index, size); - VLOG(3) << "Allocated " << p << " from system allocator."; + VLOG(10) << "Allocated " << p << " from system allocator."; if (p == nullptr) return nullptr; @@ -190,8 +191,8 @@ BuddyAllocator::PoolSet::iterator BuddyAllocator::RefillPool() { if (p == nullptr) return pool_.end(); - VLOG(3) << "Creating and inserting new block " << p - << " from system allocator"; + VLOG(10) << "Creating and inserting new block " << p + << " from system allocator"; static_cast(p)->init(cache_, MemoryBlock::FREE_CHUNK, index, max_chunk_size_, nullptr, nullptr); @@ -235,19 +236,19 @@ void* BuddyAllocator::SplitToAlloc(BuddyAllocator::PoolSet::iterator it, auto block = static_cast(std::get<2>(*it)); pool_.erase(it); - VLOG(3) << "Split block (" << block << ", " << block->total_size(cache_) - << ") into"; + VLOG(10) << "Split block (" << block << ", " << block->total_size(cache_) + << ") into"; block->split(cache_, size); - VLOG(3) << "Left block (" << block << ", " << block->total_size(cache_) - << ")"; + VLOG(10) << "Left block (" << block << ", " << block->total_size(cache_) + << ")"; block->set_type(cache_, MemoryBlock::ARENA_CHUNK); // the rest of memory if exist if (block->has_right_buddy(cache_)) { if (block->right_buddy(cache_)->type(cache_) == MemoryBlock::FREE_CHUNK) { - VLOG(3) << "Insert right block (" << block->right_buddy(cache_) << ", " - << block->right_buddy(cache_)->total_size(cache_) << ")"; + VLOG(10) << "Insert right block (" << block->right_buddy(cache_) << ", " + << block->right_buddy(cache_)->total_size(cache_) << ")"; pool_.insert( IndexSizeAddress(block->right_buddy(cache_)->index(cache_), @@ -274,7 +275,7 @@ void BuddyAllocator::CleanIdleFallBackAlloc() { return; } - VLOG(3) << "Return block " << block << " to fallback allocator."; + VLOG(10) << "Return block " << block << " to fallback allocator."; system_allocator_->Free(block, max_chunk_size_, block->index(cache_)); cache_.invalidate(block); @@ -310,7 +311,7 @@ void BuddyAllocator::CleanIdleNormalAlloc() { MemoryBlock* block = static_cast(std::get<2>(*pool)); - VLOG(3) << "Return block " << block << " to base allocator."; + VLOG(10) << "Return block " << block << " to base allocator."; system_allocator_->Free(block, max_chunk_size_, block->index(cache_)); cache_.invalidate(block); diff --git a/paddle/memory/detail/meta_cache.cc b/paddle/memory/detail/meta_cache.cc index f0721c3b94b74eed3a02e4bc744c24b97ac170a9..7e2f92b00ca5d787c1114176c5dc3304ca3ebe26 100644 --- a/paddle/memory/detail/meta_cache.cc +++ b/paddle/memory/detail/meta_cache.cc @@ -30,7 +30,7 @@ Metadata MetadataCache::load(const MemoryBlock* block) { return existing_metadata->second; } else { auto* meta = reinterpret_cast(block); - VLOG(3) << "Load MetaData type=" << meta->type; + VLOG(10) << "Load MetaData type=" << meta->type; PADDLE_ASSERT(meta->check_guards()); return *reinterpret_cast(block); } diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc index 33166d9ce23a4a345fc00a65adf63281b13643c3..6b4e46f56a0c9c9836c5b353ec9c554454ab0491 100644 --- a/paddle/memory/detail/system_allocator.cc +++ b/paddle/memory/detail/system_allocator.cc @@ -41,7 +41,16 @@ void* CPUAllocator::Alloc(size_t& index, size_t size) { index = 0; // unlock memory - void* p = malloc(size); + void* p; + +#ifdef PADDLE_USE_MKLDNN + // refer to https://github.com/01org/mkl-dnn/blob/master/include/mkldnn.hpp + // memory alignment + PADDLE_ENFORCE_EQ(posix_memalign(&p, 4096ul, size), 0); +#else + PADDLE_ENFORCE_EQ(posix_memalign(&p, 32ul, size), 0); +#endif + PADDLE_ENFORCE(p, "Fail to allocate CPU memory: size = %d .", size); if (p != nullptr) { if (FLAGS_use_pinned_memory) { diff --git a/paddle/memory/memory.cc b/paddle/memory/memory.cc index 0b648642f90a09db7452cce97eb04cedfcf55f4f..5eb1c44eb6fc45db31ef44bf79e74b79193e08aa 100644 --- a/paddle/memory/memory.cc +++ b/paddle/memory/memory.cc @@ -39,15 +39,15 @@ BuddyAllocator* GetCPUBuddyAllocator() { template <> void* Alloc(platform::CPUPlace place, size_t size) { - VLOG(3) << "Allocate " << size << " bytes on " << platform::Place(place); + VLOG(10) << "Allocate " << size << " bytes on " << platform::Place(place); void* p = GetCPUBuddyAllocator()->Alloc(size); - VLOG(3) << " pointer=" << p; + VLOG(10) << " pointer=" << p; return p; } template <> void Free(platform::CPUPlace place, void* p) { - VLOG(3) << "Free pointer=" << p << " on " << platform::Place(place); + VLOG(10) << "Free pointer=" << p << " on " << platform::Place(place); GetCPUBuddyAllocator()->Free(p); } @@ -69,11 +69,12 @@ BuddyAllocator* GetGPUBuddyAllocator(int gpu_id) { platform::GpuMinChunkSize(), platform::GpuMaxChunkSize()); } - VLOG(3) << "\n\nNOTE: each GPU device use " - << FLAGS_fraction_of_gpu_memory_to_use * 100 << "% of GPU memory.\n" - << "You can set environment variable '" - << platform::kEnvFractionGpuMemoryToUse - << "' to change the fraction of GPU usage.\n\n"; + VLOG(10) << "\n\nNOTE: each GPU device use " + << FLAGS_fraction_of_gpu_memory_to_use * 100 + << "% of GPU memory.\n" + << "You can set environment variable '" + << platform::kEnvFractionGpuMemoryToUse + << "' to change the fraction of GPU usage.\n\n"; } platform::SetDeviceId(gpu_id); return as[gpu_id]; diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 4ac96f09350ff71e765e915e8f28f1f55a89652f..7e5869cf2ca30caefa9cf29358771ef7132b5ba4 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -138,9 +138,10 @@ add_subdirectory(math) add_subdirectory(nccl) set(DEPS_OPS - recurrent_op cond_op cross_entropy_op + recurrent_op + dynamic_recurrent_op softmax_with_cross_entropy_op sum_op pool_op @@ -148,11 +149,9 @@ set(DEPS_OPS conv_op nccl_op sequence_conv_op + lod_rank_table_op lstm_op) - -op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc - DEPS framework_proto tensor net_op) op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op) op_library(cross_entropy_op DEPS cross_entropy) op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax) @@ -160,12 +159,15 @@ op_library(conv_op DEPS vol2col) op_library(sum_op DEPS net_op selected_rows_functor) op_library(pool_op DEPS pooling) op_library(pool_with_index_op DEPS pooling) +op_library(lod_rank_table_op SRCS lod_rank_table_op.cc DEPS lod_rank_table) if(WITH_GPU) op_library(nccl_op DEPS nccl_common) endif() op_library(sequence_conv_op DEPS context_project) op_library(lstm_op DEPS sequence2batch lstm_compute) - +op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc + DEPS net_op tensor_array) +op_library(recurrent_op SRCS recurrent_op.cc DEPS executor) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) op_library(${src}) @@ -177,8 +179,9 @@ cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) -cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc DEPS dynamic_recurrent_op recurrent_op tensor_array) - +cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc + rnn/recurrent_op_utils.cc + DEPS dynamic_recurrent_op) if(WITH_GPU) nv_test(nccl_op_test SRCS nccl_op_test.cu DEPS nccl_op gpu_info device_context) endif() diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc index eb8bce8da70a128bd1e0d36540dce5e296540629..eaafb9ad54b371837cbc0f3268f7f2bf169e83e8 100644 --- a/paddle/operators/accuracy_op.cc +++ b/paddle/operators/accuracy_op.cc @@ -22,22 +22,35 @@ class AccuracyOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("Inference"), - "Input(Inference) of AccuracyOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Out"), + "Input (Out) of accuracy op should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Indices"), + "Input (Indices) of accuracy op should not be null."); PADDLE_ENFORCE(ctx->HasInput("Label"), - "Input(Label) of AccuracyOp should not be null."); + "Input (Label) of accuracy op should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Accuracy"), - "Output(Accuracy) of AccuracyOp should not be null."); + "Output (Accuracy) of AccuracyOp should not be null."); - auto inference_dim = ctx->GetInputDim("Inference"); + auto inference_dim = ctx->GetInputDim("Out"); auto label_dim = ctx->GetInputDim("Label"); + // Assume indices has same shape as inference, because + // it's the output of topk. - PADDLE_ENFORCE_EQ(label_dim.size(), 1, "label must be a vector"); + PADDLE_ENFORCE_EQ(label_dim.size(), 2, "label's rank must be 2."); + PADDLE_ENFORCE_EQ(label_dim[1], 1, "label's second dimension must be 1"); PADDLE_ENFORCE_EQ(inference_dim[0], label_dim[0], - "inference size must be the same as label size"); + "the inference tensor's num_rows must be" + " the same as label."); ctx->SetOutputDim("Accuracy", {1}); - ctx->ShareLoD("Inference", /*->*/ "Accuracy"); + ctx->ShareLoD("Out", /*->*/ "Accuracy"); + } + + protected: + // IndicateDataType + framework::DataType IndicateDataType( + const framework::ExecutionContext &ctx) const override { + return framework::ToDataType(ctx.Input("Out")->type()); } }; @@ -47,19 +60,24 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { // TODO(typhoonzero): support both inference value and indices. - AddInput("Inference", "topk(indices) the network output"); + AddInput("Out", "The network output of topk (inferences)"); + AddInput("Indices", "The the network output of topk (indices)"); AddInput("Label", "Label of the training data"); // TODO(typhoonzero): AddInput("Weight", ... AddOutput("Accuracy", "The accuracy of current batch"); AddComment(R"DOC( -Accuracy. It will print accuracy rate for classification. -The accuracy is: -.. math:: -accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples}) +Accuracy Operator. + +It will print accuracy rate for classification. +The accuracy is calculated as follows: + +$$accuracy = \frac{NumOfCorrectPredicts}{NumOfAllSamples}$$ + +Both the input Out and Label can carry the LoD (Level of Details) +information, or not. But the output only shares the LoD information +with the input Out(Inference). -Both the input `Inference` and `Label` can carry the LoD (Level of Details) -information, or not. But the output only shares the LoD with input `Inference`. )DOC"); } }; @@ -68,7 +86,10 @@ information, or not. But the output only shares the LoD with input `Inference`. } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(accuracy, ops::AccuracyOp, ops::AccuracyOpMaker); -REGISTER_OP_CPU_KERNEL( - accuracy, ops::AccuracyKernel, - ops::AccuracyKernel); +REGISTER_OPERATOR(accuracy, ops::AccuracyOp, ops::AccuracyOpMaker, + paddle::framework::EmptyGradOpMaker); +// FIXME(typhoonzero): types of T is for infernece data. +// label data is always int. +REGISTER_OP_CPU_KERNEL(accuracy, + ops::AccuracyKernel, + ops::AccuracyKernel); diff --git a/paddle/operators/accuracy_op.cu b/paddle/operators/accuracy_op.cu index be58dfbd0305ba14488c2494f82a41ab6c0e8c19..a0483f367e1e3bcf4ad50ee9155f25461c1b4ed2 100644 --- a/paddle/operators/accuracy_op.cu +++ b/paddle/operators/accuracy_op.cu @@ -21,9 +21,10 @@ namespace paddle { namespace operators { using platform::PADDLE_CUDA_NUM_THREADS; -template -__global__ void AccuracyCudaKernel(const int N, const int D, const T* Xdata, - const T* labeldata, float* accuracy) { +template +__global__ void AccuracyCudaKernel(const int N, const int D, + const int64_t* Xdata, + const int64_t* labeldata, float* accuracy) { int count = 0; __shared__ int total[BlockSize]; @@ -52,13 +53,14 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), "It must use GPUPlace."); - auto* inference = ctx.Input("Inference"); + auto* inference = ctx.Input("Out"); + auto* indices = ctx.Input("Indices"); auto* label = ctx.Input("Label"); auto* accuracy = ctx.Output("Accuracy"); // FIXME(typhoonzero): only support indices currently // if add support for output values, how to detect the data type? - const T* inference_data = inference->data(); - const T* label_data = label->data(); + const int64_t* indices_data = indices->data(); + const int64_t* label_data = label->data(); float* accuracy_data = accuracy->mutable_data(ctx.GetPlace()); size_t num_samples = inference->dims()[0]; @@ -69,11 +71,11 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { return; } - AccuracyCudaKernel<<< + AccuracyCudaKernel<<< 1, PADDLE_CUDA_NUM_THREADS, 0, reinterpret_cast( ctx.device_context()) - .stream()>>>(num_samples, infer_width, inference_data, label_data, + .stream()>>>(num_samples, infer_width, indices_data, label_data, accuracy_data); } }; @@ -81,5 +83,7 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel, - paddle::operators::AccuracyOpCUDAKernel); +// FIXME(typhoonzero): types of T is for infernece data. +// label data is always int +REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel, + paddle::operators::AccuracyOpCUDAKernel); diff --git a/paddle/operators/accuracy_op.h b/paddle/operators/accuracy_op.h index 12c6b9aac8819caedbc02017cee81b37322bb72a..1968b53d19acfddaa514eca6e24d98a298d8d311 100644 --- a/paddle/operators/accuracy_op.h +++ b/paddle/operators/accuracy_op.h @@ -38,14 +38,15 @@ template class AccuracyKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto* inference = ctx.Input("Inference"); + auto* inference = ctx.Input("Out"); + auto* indices = ctx.Input("Indices"); auto* label = ctx.Input("Label"); auto* accuracy = ctx.Output("Accuracy"); float* accuracy_data = accuracy->mutable_data(ctx.GetPlace()); - const T* inference_data = inference->data(); - const T* label_data = label->data(); + const int64_t* indices_data = indices->data(); + const int64_t* label_data = label->data(); size_t num_samples = inference->dims()[0]; size_t class_dim = inference->dims()[1]; @@ -60,7 +61,7 @@ class AccuracyKernel : public framework::OpKernel { for (size_t i = 0; i < num_samples; ++i) { PADDLE_ENFORCE_GE(label_data[i], 0, "label must >= 0"); for (size_t j = 0; j < class_dim; ++j) { - if (inference_data[i * class_dim + j] == label_data[i]) { + if (indices_data[i * class_dim + j] == label_data[i]) { ++num_correct; break; } diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 90f1535fcd387c34ea39d84d9c2ec78fcbc3c764..83d35a450d0e8ebf5311cdfd948b066642ccec8c 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -43,7 +43,12 @@ class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Sigmoid operator"); AddOutput("Y", "Output of Sigmoid operator"); - AddComment("Sigmoid activation operator, sigmoid = 1 / (1 + exp(-x))"); + AddComment(R"DOC( +Sigmoid Activation Operator. + +$y = 1 / (1 + e^{-x})$ + +)DOC"); } }; @@ -54,8 +59,12 @@ class LogSigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of LogSigmoid operator"); AddOutput("Y", "Output of LogSigmoid operator"); - AddComment( - "Logsigmoid activation operator, logsigmoid = log (1 / (1 + exp(-x)))"); + AddComment(R"DOC( +Logsigmoid Activation Operator. + +$y = \log(1 / (1 + e^{-x}))$ + +)DOC"); } }; @@ -65,7 +74,12 @@ class ExpOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Exp operator"); AddOutput("Y", "Output of Exp operator"); - AddComment("Exp activation operator, exp(x) = e^x"); + AddComment(R"DOC( +Exp Activation Operator. + +$y = e^x$ + +)DOC"); } }; @@ -75,7 +89,12 @@ class ReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Relu operator"); AddOutput("Y", "Output of Relu operator"); - AddComment("Relu activation operator, relu(x) = max(x, 0)"); + AddComment(R"DOC( +Relu Activation Operator. + +$y = \max(x, 0)$ + +)DOC"); } }; @@ -87,11 +106,14 @@ class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of LeakyRelu operator"); AddOutput("Y", "Output of LeakyRelu operator"); - AddComment( - "LeakyRelu activation operator, " - "leaky_relu = max(x, alpha * x)"); AddAttr("alpha", "The small negative slope") .SetDefault(static_cast(0.02f)); + AddComment(R"DOC( +LeakyRelu Activation Operator. + +$y = \max(x, \alpha * x)$ + +)DOC"); } }; @@ -103,12 +125,20 @@ class SoftShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softshrink operator"); AddOutput("Y", "Output of Softshrink operator"); - AddComment( - "Softshrink activation operator, " - "softshrink = x - lambda, if x > lambda;" - " x + lambda, if x < lambda; 0 otherwise"); AddAttr("lambda", "non-negative offset") .SetDefault(static_cast(0.5f)); + AddComment(R"DOC( +Softshrink Activation Operator. + +$$ +y = \begin{cases} + x - \lambda, \text{if } x > \lambda \\ + x + \lambda, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -118,9 +148,12 @@ class TanhOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Tanh operator"); AddOutput("Y", "Output of Tanh operator"); - AddComment( - "Tanh activation operator, tanh = (exp(x) - exp(-x)) / (exp(x) + " - "exp(-x))"); + AddComment(R"DOC( +Tanh Activation Operator. + +$$y = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ + +)DOC"); } }; @@ -131,7 +164,12 @@ class TanhShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of TanhShrink operator"); AddOutput("Y", "Output of TanhShrink operator"); - AddComment("TanhShrink activation operator, tanhshrink(x) = x - tanh(x)"); + AddComment(R"DOC( +TanhShrink Activation Operator. + +$$y = x - \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ + +)DOC"); } }; @@ -143,13 +181,20 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of HardShrink operator"); AddOutput("Y", "Output of HardShrink operator"); - AddComment( - "HardShrink activation operator, " - "hard_shrink(x) = x if x > lambda" - "hard_shrink(x) = x if x < -lambda" - "hard_shrink(x) = 0 otherwise"); AddAttr("threshold", "The value of threshold for HardShrink") .SetDefault(static_cast(0.5)); + AddComment(R"DOC( +HardShrink Activation Operator. + +$$ +y = \begin{cases} + x, \text{if } x > \lambda \\ + x, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -159,7 +204,12 @@ class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Sqrt operator"); AddOutput("Y", "Output of Sqrt operator"); - AddComment("Sqrt activation operator, sqrt(x) = x^(1/2)"); + AddComment(R"DOC( +Sqrt Activation Operator. + +$y = \sqrt{x}$ + +)DOC"); } }; @@ -169,7 +219,12 @@ class AbsOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Abs operator"); AddOutput("Y", "Output of Abs operator"); - AddComment("Abs activation operator, abs(x) = |x|"); + AddComment(R"DOC( +Abs Activation Operator. + +$y = |x|$ + +)DOC"); } }; @@ -180,7 +235,12 @@ class ReciprocalOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Reciprocal operator"); AddOutput("Y", "Output of Reciprocal operator"); - AddComment("Reciprocal activation operator, reciprocal(x) = 1 / x"); + AddComment(R"DOC( +Reciprocal Activation Operator. + +$$y = \frac{1}{x}$$ + +)DOC"); } }; @@ -190,7 +250,14 @@ class LogOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Log operator"); AddOutput("Y", "Output of Log operator"); - AddComment("Log activation operator, log(x) = natural logarithm of x"); + AddComment(R"DOC( +Log Activation Operator. + +$y = \ln(x)$ + +Natural logarithm of x. + +)DOC"); } }; @@ -200,7 +267,12 @@ class SquareOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Square operator"); AddOutput("Y", "Output of Square operator"); - AddComment("Square activation operator, square(x) = x^2"); + AddComment(R"DOC( +Square Activation Operator. + +$y = x^2$ + +)DOC"); } }; @@ -211,7 +283,12 @@ class SoftplusOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softplus operator"); AddOutput("Y", "Output of Softplus operator"); - AddComment("Softplus activation operator, softplus(x) = log(1 + exp(x))"); + AddComment(R"DOC( +Softplus Activation Operator. + +$y = \ln(1 + e^{x})$ + +)DOC"); } }; @@ -222,7 +299,12 @@ class SoftsignOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softsign operator"); AddOutput("Y", "Output of Softsign operator"); - AddComment("Softsign activation operator, softsign(x) = x / (1 + |x|)"); + AddComment(R"DOC( +Softsign Activation Operator. + +$$y = \frac{x}{1 + |x|}$$ + +)DOC"); } }; @@ -233,11 +315,16 @@ class BReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of BRelu operator"); AddOutput("Y", "Output of BRelu operator"); - AddComment("BRelu activation operator, brelu = max(min(x, t_min), t_max)"); AddAttr("t_min", "The min marginal value of BRelu") .SetDefault(static_cast(0)); AddAttr("t_max", "The max marginal value of BRelu") .SetDefault(static_cast(24)); + AddComment(R"DOC( +BRelu Activation Operator. + +$y = \max(\min(x, t_{min}), t_{max})$ + +)DOC"); } }; @@ -249,11 +336,14 @@ class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of SoftRelu operator"); AddOutput("Y", "Output of SoftRelu operator"); - AddComment( - "SoftRelu activation operator, soft_relu = log(1 + exp(max(min(x, " - "threshold), threshold)))"); AddAttr("threshold", "The threshold value of SoftRelu") .SetDefault(static_cast(40)); + AddComment(R"DOC( +SoftRelu Activation Operator. + +$y = \ln(1 + \exp(\max(\min(x, threshold), threshold))$ + +)DOC"); } }; @@ -262,19 +352,19 @@ class ELUOpMaker : public framework::OpProtoAndCheckerMaker { public: ELUOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", - "(Tensor) The input of ELU operator, it shouldn't be empty. Input " - "is flattened and treated as a 1D array."); - AddOutput("Y", - "(Tensor) The output of ELU operator. It has the same shape as " - "the input."); - AddAttr( - "alpha", "(float, default 1.0) Alpha value in the elu formulation.") - .SetDefault(static_cast(1.)); + AddInput("X", "Input of ELU operator"); + AddOutput("Y", "Output of ELU operator"); + AddAttr("alpha", "The alpha value of ELU") + .SetDefault(static_cast(1.0f)); AddComment(R"DOC( - ELU activation operator. It applies this element-wise computation on - the input: f(x) = max(0, x) + min(0, alpha * (exp(x) - 1)). - Check .. _Link: https://arxiv.org/abs/1511.07289 for more details.)DOC"); +ELU Activation Operator. + +Applies the following element-wise computation on the input according to +https://arxiv.org/abs/1511.07289. + +$y = \max(0, x) + \min(0, \alpha * (e^x - 1))$ + +)DOC"); } }; @@ -285,9 +375,14 @@ class Relu6OpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Relu6 operator"); AddOutput("Y", "Output of Relu6 operator"); - AddComment("Relu6 activation operator, relu6 = min(max(0, x), 6)"); AddAttr("threshold", "The threshold value of Relu6") .SetDefault(static_cast(6)); + AddComment(R"DOC( +Relu6 Activation Operator. + +$y = \min(\max(0, x), 6)$ + +)DOC"); } }; @@ -298,9 +393,14 @@ class PowOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Pow operator"); AddOutput("Y", "Output of Pow operator"); - AddComment("Pow activation operator, pow(x, factor) = x^factor"); AddAttr("factor", "The exponential factor of Pow") .SetDefault(static_cast(1)); + AddComment(R"DOC( +Pow Activation Operator. + +$y = x^{factor}$ + +)DOC"); } }; @@ -311,11 +411,16 @@ class STanhOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of STanh operator"); AddOutput("Y", "Output of STanh operator"); - AddComment("STanh activation operator, stanh = b * tanh(a * x)"); AddAttr("scale_a", "The scale parameter of a for the input") .SetDefault(static_cast(2 / 3)); AddAttr("scale_b", "The scale parameter of b for the input") .SetDefault(static_cast(1.7159)); + AddComment(R"DOC( +STanh Activation Operator. + +$$y = b * \frac{e^{a * x} - e^{-a * x}}{e^{a * x} + e^{-a * x}}$$ + +)DOC"); } }; @@ -327,12 +432,19 @@ class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of ThresholdedRelu operator"); AddOutput("Y", "Output of ThresholdedRelu operator"); - AddComment( - "ThresholdedRelu activation operator, " - "thresholded_relu = x for x > threshold, " - "thresholded_relu = 0 otherwise."); AddAttr("threshold", "The threshold location of activation") .SetDefault(static_cast(1.0)); + AddComment(R"DOC( +ThresholdedRelu Activation Operator. + +$$ +y = \begin{cases} + x, \text{if } x > threshold \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -344,27 +456,23 @@ class HardSigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of HardSigmoid operator"); AddOutput("Y", "Output of HardSigmoid operator"); + AddAttr("slope", "Slope for linear approximation of sigmoid") + .SetDefault(static_cast(0.2)); + AddAttr("offset", "Offset for linear approximation of sigmoid") + .SetDefault(static_cast(0.5)); AddComment(R"DOC( -Hard Sigmoid activation operator. +HardSigmoid Activation Operator. -Segment-wise linear approximation of sigmoid[1]. -This is much faster than sigmoid. +Segment-wise linear approximation of sigmoid(https://arxiv.org/abs/1603.00391), +which is much faster than sigmoid. -hard_sigmoid = max(0, min(1, slope * x + shift)) +$y = \max(0, \min(1, slope * x + shift))$ The slope should be positive. The offset can be either positive or negative. -The default slope and shift are set from [1]. +The default slope and shift are set according to the above reference. It is recommended to use the defaults for this activation. -References: - [1] Noisy Activation Functions - (https://arxiv.org/abs/1603.00391) - - )DOC"); - AddAttr("slope", "Slope for linear approximation of sigmoid") - .SetDefault(static_cast(0.2)); - AddAttr("offset", "Offset for linear approximation of sigmoid") - .SetDefault(static_cast(0.5)); +)DOC"); } }; diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index e4c6b2e09cd71f00a2ef73173205b9066c34fcf5..ceb4b4e40b67473f42e67e3f02f8e012e1b1eb50 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -232,7 +232,7 @@ struct HardShrinkGradFunctor : public BaseActivationFunctor { } }; -// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < lambda; 0 +// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < -lambda; 0 // otherwise template struct SoftShrinkFunctor : public BaseActivationFunctor { @@ -547,6 +547,7 @@ struct ELUGradFunctor : public BaseActivationFunctor { } }; +// FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5198 template struct PowFunctor : public BaseActivationFunctor { float factor; diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc index 24e419b532d97bc16ab96dad418d6e73c03f30a0..b717e1647e4b89285b841420650dc69e8a1e0c58 100644 --- a/paddle/operators/adadelta_op.cc +++ b/paddle/operators/adadelta_op.cc @@ -64,16 +64,15 @@ class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Param", "(Tensor) Input parameter"); AddInput("Grad", "(Tensor) Input gradient"); - AddInput("AvgSquaredGrad", - "(Tensor) Input expectation of squared gradient"); + AddInput("AvgSquaredGrad", "(Tensor) Input average of squared gradient"); AddInput("AvgSquaredUpdate", - "(Tensor) Input expectation of squared parameter updates"); + "(Tensor) Input average of squared parameter updates"); AddOutput("ParamOut", "(Tensor) Output parameter"); AddOutput("AvgSquaredGradOut", - "(Tensor) Output expectation of squared gradient"); + "(Tensor) Output average of squared gradient"); AddOutput("AvgSquaredUpdateOut", - "(Tensor) Output expectation of squared parameter updates"); + "(Tensor) Output average of squared parameter updates"); AddAttr("rho", "(float, default 0.95) Exponential decay rate " @@ -84,22 +83,21 @@ class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker { "numerical stability") .SetDefault(1.0e-6f); AddComment(R"DOC( -Adadelta Updates Operator. +Adadelta Optimizer. -This implements the Adadelta optimizer[1]. Adadelta is a per-dimension -adaptive learning rate method for gradient descent. +Adadelta optimizer is implemented as explained in: +https://arxiv.org/abs/1212.5701 +Adadelta is a per-dimension adaptive learning rate method used +for gradient descent. -Adadelta updates: +Adadelta updates are as follows: -avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * grad * grad -param_update = - sqrt((avg_squared_update + epsilon) / - (avg_squared_grad_out + epsilon)) * grad -avg_squared_update_out = rho * avg_squared_update + (1 - rho) * param_update**2 -param_out = param + param_update - -References: - [1] ADADELTA: An Adaptive Learning Rate Method - https://arxiv.org/abs/1212.5701 +$$avgSquaredGradOut = \rho * avgSquaredGrad + (1 - \rho) * grad * grad \break +paramUpdate = - $\sqrt{((avgSquaredUpdate + \epsilon) / + (avgSquaredGrad_out + \epsilon))}$ * grad \break +avgSquaredUpdateOut = \rho * avgSquaredUpdate + (1 - \rho) * + {(paramUpdate)}^2 \break +paramOut = param + paramUpdate$$ )DOC"); } diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index bc081f87dcab0dcd8ef329dcb1f66b627c82b4a2..8d1a2b7938d2c6607cbeb3cecb72d1d5b83dd8b9 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -73,12 +73,16 @@ class AdagradOpMaker : public framework::OpProtoAndCheckerMaker { Adaptive Gradient Algorithm (Adagrad). -moment_out = moment + grad * grad -param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon) +The update is done as follows: + +$$momentOut = moment + grad * grad \break +paramOut = param - learningRate * grad / ($\sqrt{momentOut}$ + \epsilon) \break +$$ The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) -does not have the epsilon attribute. It is added here for numerical stability -by avoiding division by zero. +does not have the epsilon attribute. It is added here in our implementation +as also proposed here: http://cs231n.github.io/neural-networks-3/#ada +for numerical stability to avoid the division by zero error. )DOC"); } diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc index 3572de06bd60f7979e3bfbf39856b04942ce81c0..97a091ae766abfba5412bbd32c34a6f80701fbf7 100644 --- a/paddle/operators/adam_op.cc +++ b/paddle/operators/adam_op.cc @@ -51,8 +51,8 @@ class AdamOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1, "Beta1 power accumulator should have 1 dimension"); auto beta2_pow_dims = ctx->GetInputDim("Beta2Pow"); - PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1, - "Beta1 power accumulator should have 1 dimension"); + PADDLE_ENFORCE_EQ(framework::product(beta2_pow_dims), 1, + "Beta2 power accumulator should have 1 dimension"); auto param_dims = ctx->GetInputDim("Param"); PADDLE_ENFORCE_EQ( @@ -60,10 +60,10 @@ class AdamOp : public framework::OperatorWithKernel { "Param and Grad input of AdamOp should have same dimension"); PADDLE_ENFORCE_EQ( param_dims, ctx->GetInputDim("Moment1"), - "Param and Moment input of AdamOp should have same dimension"); + "Param and Moment1 input of AdamOp should have same dimension"); PADDLE_ENFORCE_EQ( param_dims, ctx->GetInputDim("Moment2"), - "Param and InfNorm input of AdamOp should have same dimension"); + "Param and Moment2 input of AdamOp should have same dimension"); ctx->SetOutputDim("ParamOut", param_dims); ctx->SetOutputDim("Moment1Out", param_dims); @@ -103,23 +103,20 @@ class AdamOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(1.0e-8f); AddComment(R"DOC( -Adam Updates Operator. +Adam Optimizer. This implements the Adam optimizer from Section 2 of the Adam -paper[1]. Adam is a first-order gradient-based optimization -method based on adaptive estimates of lower-order moments. +paper : https://arxiv.org/abs/1412.6980. +Adam is a first-order gradient-based optimization method based on +adaptive estimates of lower-order moments. Adam updates: -moment1_out = beta1 * moment1 + (1 − beta1) * grad -moment2_out = beta2 * moment2 + (1 − beta2) * grad * grad -learning_rate_t = learning_rate_t * - sqrt(1 - beta2_pow) / (1 - beta1_pow) -param_out = param - learning_rate_t * moment1/ (sqrt(moment2) + epsilon) - -References: - [1] Adam: A Method for Stochastic Optimization - (https://arxiv.org/abs/1412.6980) +$$moment_1_{out} = \beta_1 * moment_1 + (1 - \beta_1) * grad \break +moment_2_{out} = \beta_2 * moment_2 + (1 - \beta_2) * grad * grad \break +learningRate = learningRate * + $\sqrt{(1 - \beta_2_{pow})}$ / (1 - \beta_1_{pow}) \break +paramOut = param - learningRate * moment_1/ ($\sqrt{(moment_2)} + \epsilon)$$ )DOC"); } diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc index ff2565774115571166712b03c8990e5bf8de12a5..14cf3841b33a8153549e4c99ed2b75286e9c64db 100644 --- a/paddle/operators/adamax_op.cc +++ b/paddle/operators/adamax_op.cc @@ -99,26 +99,22 @@ class AdamaxOpMaker : public framework::OpProtoAndCheckerMaker { "Constant for numerical stability") .SetDefault(1.0e-8f); AddComment(R"DOC( -Adamax Updates Operator. +Adamax Optimizer. -This implements the Adamax optimizer from Section 7 of the Adam -paper[1]. Adamax is a variant of the +We implement the Adamax optimizer from Section 7 of the Adam +paper: https://arxiv.org/abs/1412.6980. Adamax is a variant of the Adam algorithm based on the infinity norm. Adamax updates: -moment_out = beta1 * moment + (1 - beta1) * grad -inf_norm_out = max(beta2 * inf_norm + epsilon, abs(grad)) -learning_rate_t = learning_rate/(1 - beta1_pow) -param_out = param - learning_rate_t * moment_out/inf_norm_out +$$momentOut = \beta_1 * moment + (1 - \beta_1) * grad \break +infNormOut = max(\beta_2 * infNorm + \epsilon, |grad|) \break +learningRate = learningRate /(1 - \beta_1_{pow}) \break +paramOut = param - learningRate * momentPut / infNormOut$$ The original paper does not have an epsilon attribute. -However, it is added here for numerical stability -by preventing divide by 0. - -References: - [1] Adam: A Method for Stochastic Optimization - (https://arxiv.org/abs/1412.6980) +However, it is added here for numerical stability to prevent the +division by 0 error. )DOC"); } diff --git a/paddle/operators/auc_op.cc b/paddle/operators/auc_op.cc index cf3dbc5d10c66cbb344ca8cf8c46432eabef4a07..ccb969ab23a8e1df713278513a66b10f21690108 100644 --- a/paddle/operators/auc_op.cc +++ b/paddle/operators/auc_op.cc @@ -23,18 +23,26 @@ class AucOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContext *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("Inference"), - "Input of Inference must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("Out"), "Input of Out should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Indices"), + "Input of Indices should not be null."); PADDLE_ENFORCE(ctx->HasInput("Label"), - "Input of Label must be initialized."); - auto inference_dim = ctx->GetInputDim("Inference"); - auto label_dim = ctx->GetInputDim("Label"); + "Input of Label should not be null."); + auto inference_height = ctx->GetInputDim("Out")[0]; + auto label_height = ctx->GetInputDim("Label")[0]; - PADDLE_ENFORCE_EQ(inference_dim, label_dim, - "inference and label should have same shape"); + PADDLE_ENFORCE_EQ(inference_height, label_height, + "Out and Label should have same height."); ctx->SetOutputDim("AUC", {1}); - ctx->ShareLoD("Inference", /*->*/ "AUC"); + ctx->ShareLoD("Out", /*->*/ "AUC"); + } + + protected: + // IndicateDataType + framework::DataType IndicateDataType( + const framework::ExecutionContext &ctx) const override { + return framework::ToDataType(ctx.Input("Out")->type()); } }; @@ -42,16 +50,22 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker { public: AucOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Inference", - "A floating point tensor of arbitrary shape and whose values" - "are in the range [0, 1]."); + AddInput("Out", + "A floating point 2D tensor, values are in the range [0, 1]." + "Each row is sorted in descending order. This input should be the" + "output of topk." + "Typically, this tensor indicates the probability of each label"); + AddInput("Indices", + "An int 2D tensor, indicating the indices of original" + "tensor before sorting. Typically, this tensor indicates which " + "label the probability stands for."); AddInput("Label", - "A tensor whose shape matches " - "Inference. Will be cast to bool."); + "A 2D int tensor indicating the label of the training data." + "The height is batch size and width is always 1."); // TODO(typhoonzero): support weight input AddOutput("AUC", "A scalar representing the " - "current area-under-curve."); + "current area-under-the-curve."); AddAttr("curve", "Curve type, can be 'ROC' or 'PR'.") .SetDefault("ROC"); @@ -60,19 +74,18 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker { " roc curve.") .SetDefault(200); - AddComment( - R"DOC(Computes the AUC according forward output and label. -Best to use for binary classification evaluations. + AddComment(R"DOC( +Area Under The Curve (AUC) Operator. +This implementation computes the AUC according to forward output and label. +It is used very widely in binary classification evaluation. As a note: If input label contains values other than 0 and 1, it will be cast -to bool. - -You can find the definations here: +to bool. You can find the relevant definitions here: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve -Possible curves are: -- ROC: Receiver operating characteristic -- PR: Precision Recall +There are two types of possible curves: +1. ROC: Receiver operating characteristic +2. PR: Precision Recall )DOC"); } }; diff --git a/paddle/operators/auc_op.h b/paddle/operators/auc_op.h index be6ef29d5f6cff5b9ebdf7d8564b2e2792c3b5cb..e5ac57b038ac32ed35bce35e477ede0cdb5da813 100644 --- a/paddle/operators/auc_op.h +++ b/paddle/operators/auc_op.h @@ -29,7 +29,7 @@ template class AucKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto* inference = ctx.Input("Inference"); + auto* inference = ctx.Input("Out"); auto* label = ctx.Input("Label"); auto* auc = ctx.Output("AUC"); @@ -46,18 +46,11 @@ class AucKernel : public framework::OpKernel { thresholds_list[0] = 0.0f - kEpsilon; thresholds_list[num_thresholds - 1] = 1.0f + kEpsilon; - size_t num_samples = inference->numel(); + size_t batch_size = inference->dims()[0]; + size_t inference_width = inference->dims()[1]; const T* inference_data = inference->data(); - Tensor label_casted; - label_casted.Resize(label->dims()); - bool* label_casted_data = label_casted.mutable_data(ctx.GetPlace()); - - const int* label_data = label->data(); - // cast label_data to bool - for (size_t i = 0; i < num_samples; i++) { - label_casted_data[i] = static_cast(label_data[i]); - } + const int64_t* label_data = label->data(); // Create local tensor for storing the curve: TP, FN, TN, FP // TODO(typhoonzero): use eigen op to caculate these values. @@ -68,23 +61,27 @@ class AucKernel : public framework::OpKernel { true_negative.Resize({num_thresholds}); false_positive.Resize({num_thresholds}); - int* tp_data = true_positive.mutable_data(ctx.GetPlace()); - int* fn_data = false_negative.mutable_data(ctx.GetPlace()); - int* tn_data = true_negative.mutable_data(ctx.GetPlace()); - int* fp_data = false_positive.mutable_data(ctx.GetPlace()); + int64_t* tp_data = true_positive.mutable_data(ctx.GetPlace()); + int64_t* fn_data = false_negative.mutable_data(ctx.GetPlace()); + int64_t* tn_data = true_negative.mutable_data(ctx.GetPlace()); + int64_t* fp_data = false_positive.mutable_data(ctx.GetPlace()); for (int idx_thresh = 0; idx_thresh < num_thresholds; idx_thresh++) { // caculate TP, FN, TN, FP for current thresh - int tp = 0, fn = 0, tn = 0, fp = 0; - for (size_t i = 0; i < num_samples; i++) { - if (label_casted_data[i]) { - if (inference_data[i] >= (thresholds_list[idx_thresh])) { + int64_t tp = 0, fn = 0, tn = 0, fp = 0; + for (size_t i = 0; i < batch_size; i++) { + // NOTE: label_data used as bool, labels >0 will be treated as true. + if (label_data[i]) { + // use first(max) data in each row + if (inference_data[i * inference_width] >= + (thresholds_list[idx_thresh])) { tp++; } else { fn++; } } else { - if (inference_data[i] >= (thresholds_list[idx_thresh])) { + if (inference_data[i * inference_width] >= + (thresholds_list[idx_thresh])) { fp++; } else { tn++; diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index f2c8be4c54eed9cd0aeb004eeb74a42adc0695f5..7d73dfde786208fe217dac97325d432fb80052ad 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -51,6 +51,10 @@ class BatchNormOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("SavedMean"), ""); PADDLE_ENFORCE(ctx->HasOutput("SavedVariance"), ""); + const float epsilon = ctx->Attrs().Get("epsilon"); + PADDLE_ENFORCE_GE(epsilon, 0.0, "epsilon should be larger than 0"); + PADDLE_ENFORCE_LE(epsilon, 0.001, "epsilon should not be too large"); + // make sure Mean/MeanOut and Variance/VarianceOut share memory in Python PADDLE_ENFORCE_EQ(ctx->Inputs("Mean")[0], ctx->Outputs("MeanOut")[0], "Mean and MeanOut should share the same memory"); @@ -66,7 +70,7 @@ class BatchNormOp : public framework::OperatorWithKernel { : x_dims[x_dims.size() - 1]); PADDLE_ENFORCE(x_dims.size() >= 3 && x_dims.size() <= 5, - "Input x must have 3 to 5 dimensions."); + "Input X must have 3 to 5 dimensions."); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale").size(), 1UL); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale")[0], C); @@ -93,16 +97,16 @@ class BatchNormOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "The input tensor"); AddInput("Scale", "Scale is a 1-dimensional tensor of size C " - "to be applied to the output"); + "that is applied to the output"); AddInput("Bias", "Bias is a 1-dimensional tensor of size C " - "to be applied to the output"); + "that is applied to the output"); AddInput("Mean", - "The global mean (for training) or the " + "The global mean (for training) or " "estimated mean (for testing)"); AddInput("Variance", "The global variance (for training) " - "or the estimated Variance (for testing)"); + "or estimated Variance (for testing)"); AddOutput("Y", "result after normalization"); AddOutput("MeanOut", "Share memory with Mean. " @@ -119,10 +123,14 @@ class BatchNormOpMaker : public framework::OpProtoAndCheckerMaker { "will apply to output when training") .AsIntermediate(); AddComment(R"DOC( -https://arxiv.org/pdf/1502.03167.pdf +Batch Normalization. -NHWC `[batch, in_height, in_width, in_channels]` -NCHW `[batch, in_channels, in_height, in_width]` +Batch Norm has been implemented as discussed in the paper: +https://arxiv.org/pdf/1502.03167.pdf +Can be used as a normalizer function for conv2d and fully_connected operations. +The required data format for this layer is one of the following: +1. NHWC `[batch, in_height, in_width, in_channels]` +2. NCHW `[batch, in_channels, in_height, in_width]` )DOC"); } @@ -297,7 +305,6 @@ class BatchNormGradOp : public framework::OperatorWithKernel { framework::DataType IndicateDataType( const framework::ExecutionContext &ctx) const override { - VLOG(3) << "IndicateDataType " << this->Type(); const auto *var = ctx.InputVar(framework::GradVarName("Y")); if (var == nullptr) { PADDLE_THROW("can't find Y@GRAD"); diff --git a/paddle/operators/cast_op.cc b/paddle/operators/cast_op.cc index 19187894c3f4803ef241d5e0c159852c0d9687da..70ee7861bab3a982eae60dd85b10c2e41f5827d0 100644 --- a/paddle/operators/cast_op.cc +++ b/paddle/operators/cast_op.cc @@ -23,13 +23,17 @@ class CastOpProtoMaker : public framework::OpProtoAndCheckerMaker { CastOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensor of cast op"); - AddOutput("Out", "the output tensor of cast op"); - AddComment(R"DOC(Cast operator. -cast the input tensor to other data type. -)DOC"); + AddInput("X", "The input tensor of cast op"); + AddOutput("Out", "The output tensor of cast op"); AddAttr("out_data_type", "output data type"); AddAttr("in_data_type", "input data type"); + AddComment(R"DOC( +Cast Operator. + +This Operator casts the input tensor to another data type and +returns tha Output Tensor. + +)DOC"); } }; diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc index f80204c6833d6436f2cf21610beea45b36787eea..3e9066ceb2a4a4dc19fdf5ef02bb7fadaab4bfff 100644 --- a/paddle/operators/clip_op.cc +++ b/paddle/operators/clip_op.cc @@ -49,8 +49,11 @@ class ClipOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr( "max", "(float)Maximum value, above which element is replaced by max"); AddComment(R"DOC( -Clip operator limits the given input within an interval. The interval is +Clip Operator. + +The clip operator limits the value of given input within an interval. The interval is specified with arguments 'min' and 'max'. + )DOC"); } }; diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc index e11e51b4583817ef50cd447dbcf4c7202a152422..5f052689251bc023df635d41c1e64a660a0aa488 100644 --- a/paddle/operators/concat_op.cc +++ b/paddle/operators/concat_op.cc @@ -56,20 +56,24 @@ class ConcatOpMaker : public framework::OpProtoAndCheckerMaker { public: ConcatOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of concat operator.").AsDuplicable(); - AddOutput("Out", "the output tensor of concat operator."); - AddComment(R"DOC( - Join the input tensors along with the axis. - Examples: - Input[0] = [[1,2],[3,4]] - Input[1] = [[5,6]] - axis = 0 - Output = [[1,2], - [3,4], - [5,6]] - )DOC"); - AddAttr("axis", "The axis which the inputs will be joined with.") + AddInput("X", "Input tensors of concat operator.").AsDuplicable(); + AddOutput("Out", "Output tensor of concat operator."); + AddAttr("axis", + "The axis along which the input tensors will be concatenated.") .SetDefault(0); + AddComment(R"DOC( +Concat Operator. + +Concatenate the input tensors along dimension axis. +Examples: + Input[0] = [[1,2],[3,4]] + Input[1] = [[5,6]] + axis = 0 + Output = [[1,2], + [3,4], + [5,6]] + +)DOC"); } }; diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index adcd867f502d166f851926fde602dbb3fed9b48e..b809bdc3a0fea727f2fb6ea0a55672ee9b0bbd04 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -216,11 +216,12 @@ class CondOpProtoAndCheckerMaker : public framework::OpProtoAndCheckerMaker { AddOutput("IndexTensors", "Index Tensors contains indices for true/false"); AddComment(R"DOC( -Sample dependent Cond Operator: -Given Cond[i] as a 1/0 vector to indicate true/false -The equation is: -Out[i] = subnet_t[i], if Cond[i] == true -Out[i] = subnet_t[i], if Cond[i] == false +Sample Dependent Conditional Operator. + +Given Cond[i] as a 1/0 vector to indicate true/false: +Out[i] = subnet_true[i], if Cond[i] == true +Out[i] = subnet_false[i], if Cond[i] == false + )DOC"); } }; diff --git a/paddle/operators/conv2d_transpose_cudnn_op.cc b/paddle/operators/conv2d_transpose_cudnn_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..8ce94e0f04f14e1eae7e7d01280601cc72dea8c4 --- /dev/null +++ b/paddle/operators/conv2d_transpose_cudnn_op.cc @@ -0,0 +1,50 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/conv2d_transpose_op.h" + +namespace paddle { +namespace operators { + +class CudnnConv2DTransposeOpMaker : public Conv2DTransposeOpMaker { + public: + CudnnConv2DTransposeOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : Conv2DTransposeOpMaker(proto, op_checker) { + AddAttr>("dilations", "dilations of convolution operator.") + .SetDefault(std::vector{1, 1}); + AddAttr("workspace_size_MB", + "workspace size for cudnn, in MB, " + "workspace is a section of GPU memory which will be " + "allocated/freed each time the operator runs, larger " + "workspace size can increase performance but also requires " + "better hardward. This size should be carefully setted.") + .SetDefault(4096); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(conv2d_transpose_cudnn, ops::Conv2DTransposeOp, + ops::CudnnConv2DTransposeOpMaker, conv2d_transpose_cudnn_grad, + ops::Conv2DTransposeOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv2d_transpose_cudnn, + ops::GemmConv2DTransposeKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_transpose_cudnn_grad, + ops::GemmConv2DTransposeGradKernel); diff --git a/paddle/operators/conv2d_transpose_cudnn_op.cu b/paddle/operators/conv2d_transpose_cudnn_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..61fcfb3bd8fa57f2c45fbf3a980dbe41041cff18 --- /dev/null +++ b/paddle/operators/conv2d_transpose_cudnn_op.cu @@ -0,0 +1,240 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memory.h" +#include "paddle/operators/conv2d_transpose_op.h" +#include "paddle/platform/assert.h" +#include "paddle/platform/cudnn_helper.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using ScopedTensorDescriptor = platform::ScopedTensorDescriptor; +using ScopedFilterDescriptor = platform::ScopedFilterDescriptor; +using ScopedConvolutionDescriptor = platform::ScopedConvolutionDescriptor; +using DataLayout = platform::DataLayout; +using CUDADeviceContext = platform::CUDADeviceContext; + +static constexpr size_t kConvCudnnWorkspaceLimitBytes = 1024 * 1024 * 1024; + +template +class CudnnConvTransposeOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + auto* input = ctx.Input("Input"); + auto* filter = ctx.Input("Filter"); + auto* output = ctx.Output("Output"); + + std::vector strides = ctx.Attr>("strides"); + std::vector paddings = ctx.Attr>("paddings"); + // cudnn v5 does not support dilations + std::vector dilations = ctx.Attr>("dilations"); + int user_workspace_size = ctx.Attr("workspace_size_MB"); + + const T* input_data = input->data(); + const T* filter_data = filter->data(); + T* output_data = output->mutable_data(ctx.GetPlace()); + // ------------------- cudnn descriptors --------------------- + ScopedTensorDescriptor input_desc; + ScopedTensorDescriptor output_desc; + ScopedFilterDescriptor filter_desc; + ScopedConvolutionDescriptor conv_desc; + DataLayout layout = DataLayout::kNCHW; + + // N, M, H, W + cudnnTensorDescriptor_t cudnn_input_desc = input_desc.descriptor( + layout, framework::vectorize2int(input->dims())); + // N, C, O_h, O_w + cudnnTensorDescriptor_t cudnn_output_desc = output_desc.descriptor( + layout, framework::vectorize2int(output->dims())); + // M, C, K_h, K_w + cudnnFilterDescriptor_t cudnn_filter_desc = filter_desc.descriptor( + layout, framework::vectorize2int(filter->dims())); + cudnnConvolutionDescriptor_t cudnn_conv_desc = + conv_desc.descriptor(paddings, strides, dilations); + + // ------------------- cudnn conv workspace --------------------- + void* cudnn_workspace = nullptr; + size_t workspace_size_in_bytes; // final workspace to allocate. + size_t workspace_size_limit = kConvCudnnWorkspaceLimitBytes; + if (user_workspace_size > 0) { + workspace_size_limit = user_workspace_size * 1024 * 1024; + } + // ------------------- cudnn conv algorithm --------------------- + cudnnConvolutionBwdDataAlgo_t algo; + auto handle = ctx.cuda_device_context().cudnn_handle(); + // Get the algorithm + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionBackwardDataAlgorithm( + handle, cudnn_filter_desc, cudnn_input_desc, cudnn_conv_desc, + // dxDesc: Handle to the previously initialized output tensor + // descriptor. + cudnn_output_desc, CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &algo)); + + // get workspace size able to allocate + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardDataWorkspaceSize( + handle, cudnn_filter_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_output_desc, algo, &workspace_size_in_bytes)); + + // Allocate on GPU memory + platform::GPUPlace gpu = boost::get(ctx.GetPlace()); + cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes); + + // ------------------- cudnn conv transpose forward --------------------- + T alpha = 1.0f, beta = 0.0f; + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardData( + handle, &alpha, cudnn_filter_desc, filter_data, cudnn_input_desc, + input_data, cudnn_conv_desc, algo, cudnn_workspace, + workspace_size_in_bytes, &beta, cudnn_output_desc, output_data)); + + // Release the cudnn workspace + paddle::memory::Free(gpu, cudnn_workspace); + } +}; + +template +class CudnnConvTransposeGradOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + auto input = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto output_grad = ctx.Input(framework::GradVarName("Output")); + auto input_grad = ctx.Output(framework::GradVarName("Input")); + auto filter_grad = ctx.Output(framework::GradVarName("Filter")); + const T* input_data = input->data(); + const T* output_grad_data = output_grad->data(); + const T* filter_data = filter->data(); + + std::vector strides = ctx.Attr>("strides"); + std::vector paddings = ctx.Attr>("paddings"); + // cudnn v5 does not support dilations + std::vector dilations = ctx.Attr>("dilations"); + int user_workspace_size = ctx.Attr("workspace_size_MB"); + + // ------------------- cudnn descriptors --------------------- + ScopedTensorDescriptor input_desc; + ScopedTensorDescriptor output_desc; + ScopedFilterDescriptor filter_desc; + ScopedConvolutionDescriptor conv_desc; + DataLayout layout = DataLayout::kNCHW; + + // Input: (N, M, H, W) + cudnnTensorDescriptor_t cudnn_input_desc = input_desc.descriptor( + layout, framework::vectorize2int(input->dims())); + // Output: (N, C, O_H, O_W) + cudnnTensorDescriptor_t cudnn_output_desc = output_desc.descriptor( + layout, framework::vectorize2int(output_grad->dims())); + // Filter (M, C, K_H, K_W) + cudnnFilterDescriptor_t cudnn_filter_desc = filter_desc.descriptor( + layout, framework::vectorize2int(filter->dims())); + + cudnnConvolutionDescriptor_t cudnn_conv_desc = + conv_desc.descriptor(paddings, strides, dilations); + + // ------------------- cudnn backward algorithm --------------------- + cudnnConvolutionFwdAlgo_t data_algo; + cudnnConvolutionBwdFilterAlgo_t filter_algo; + size_t bwd_filter_ws_size, fwd_ws_size; + size_t workspace_size_in_bytes = 0; + size_t workspace_size_limit = kConvCudnnWorkspaceLimitBytes; + if (user_workspace_size > 0) { + workspace_size_limit = user_workspace_size * 1024 * 1024; + } + + auto handle = ctx.cuda_device_context().cudnn_handle(); + if (input_grad) { + // choose backward algorithm for data + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( + handle, cudnn_output_desc, cudnn_filter_desc, cudnn_conv_desc, + cudnn_input_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &data_algo)); + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardWorkspaceSize( + handle, cudnn_output_desc, cudnn_filter_desc, cudnn_conv_desc, + cudnn_input_desc, data_algo, &fwd_ws_size)); + workspace_size_in_bytes = std::max(workspace_size_in_bytes, fwd_ws_size); + } + + if (filter_grad) { + // choose backward algorithm for filter + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardFilterAlgorithm( + handle, cudnn_output_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_filter_desc, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &filter_algo)); + + // get workspace for backwards filter algorithm + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardFilterWorkspaceSize( + handle, cudnn_output_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_filter_desc, filter_algo, &bwd_filter_ws_size)); + workspace_size_in_bytes = + std::max(workspace_size_in_bytes, bwd_filter_ws_size); + } + + // ------------------- cudnn conv workspace --------------------- + // Already on GPU + void* cudnn_workspace = nullptr; + platform::GPUPlace gpu = boost::get(ctx.GetPlace()); + cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes); + // ------------------- cudnn conv backward data --------------------- + // FIXME(typhoonzero): template type T may not be the same as cudnn call. + T alpha = 1.0f, beta = 0.0f; + if (input_grad) { + T* input_grad_data = input_grad->mutable_data(ctx.GetPlace()); + auto t = framework::EigenVector::Flatten(*input_grad); + t.device(ctx.GetEigenDevice()) = + t.constant(static_cast(0)); + + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionForward( + handle, &alpha, cudnn_output_desc, output_grad_data, + cudnn_filter_desc, filter_data, cudnn_conv_desc, data_algo, + cudnn_workspace, workspace_size_in_bytes, &beta, cudnn_input_desc, + input_grad_data)); + } + + // ------------------- cudnn conv backward filter --------------------- + if (filter_grad) { + T* filter_grad_data = filter_grad->mutable_data(ctx.GetPlace()); + auto t = framework::EigenVector::Flatten(*filter_grad); + t.device(ctx.GetEigenDevice()) = + t.constant(static_cast(0)); + // Gradient with respect to the filter + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardFilter( + handle, &alpha, cudnn_output_desc, output_grad_data, cudnn_input_desc, + input_data, cudnn_conv_desc, filter_algo, cudnn_workspace, + workspace_size_in_bytes, &beta, cudnn_filter_desc, filter_grad_data)); + } + // Release the cudnn workspace + paddle::memory::Free(gpu, cudnn_workspace); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn, + ops::CudnnConvTransposeOpKernel); +REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn_grad, + ops::CudnnConvTransposeGradOpKernel); diff --git a/paddle/operators/conv2dtranspose_op.cc b/paddle/operators/conv2d_transpose_op.cc similarity index 85% rename from paddle/operators/conv2dtranspose_op.cc rename to paddle/operators/conv2d_transpose_op.cc index c1b231906e2f172b6f9cee55f850d1a5ec6c3221..8f5d18cddf45d1129040454adbc95a511ccf0583 100644 --- a/paddle/operators/conv2dtranspose_op.cc +++ b/paddle/operators/conv2d_transpose_op.cc @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/conv2dtranspose_op.h" +#include "paddle/operators/conv2d_transpose_op.h" namespace paddle { namespace operators { @@ -54,15 +54,16 @@ Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( AddInput( "Input", "(Tensor) The input tensor of convolution transpose operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of input channels, H and W is the height and width of image."); + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of input channels, H is the height of the image, and " + "W is the width of the image."); AddInput("Filter", "(Tensor) The filter tensor of convolution transpose operator." "The format of the filter tensor is CMHW, where C is the number of " "output image channels, M is the number of input image channels, " - "H and W is height and width of filter. " + "H is the height of the filter, and W is the width of the filter. " "We enforce groups number == 1 and padding == 0 in " - "convolution transpose Scenario."); + "the convolution transpose scenario."); AddOutput("Output", "(Tensor) The output tensor of convolution transpose operator." "The format of output tensor is also NCHW."); @@ -73,9 +74,12 @@ Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( "paddings of convolution transpose operator.") .SetDefault({0, 0}); AddComment(R"DOC( -The convolution transpose operation calculates the output based on the input, filter -and strides, paddings, groups parameters. The size of each dimension of the -parameters is checked in the infer-shape. +Convolution Transpose Operator. + +The convolution transpose operation calculates the output based on the input, +filter, strides, paddings, and groups parameters. The size of each dimension +of the parameters is checked in the infer-shape method. + )DOC"); } @@ -95,13 +99,13 @@ void Conv2DTransposeOpGrad::InferShape( } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(conv2dtranspose, ops::Conv2DTransposeOp, - ops::Conv2DTransposeOpMaker, conv2dtranspose_grad, +REGISTER_OP(conv2d_transpose, ops::Conv2DTransposeOp, + ops::Conv2DTransposeOpMaker, conv2d_transpose_grad, ops::Conv2DTransposeOpGrad); REGISTER_OP_CPU_KERNEL( - conv2dtranspose, + conv2d_transpose, ops::GemmConv2DTransposeKernel); REGISTER_OP_CPU_KERNEL( - conv2dtranspose_grad, + conv2d_transpose_grad, ops::GemmConv2DTransposeGradKernel); diff --git a/paddle/operators/conv2dtranspose_op.cu b/paddle/operators/conv2d_transpose_op.cu similarity index 89% rename from paddle/operators/conv2dtranspose_op.cu rename to paddle/operators/conv2d_transpose_op.cu index 761bc1959e69be94f43571728e6b92a322558b99..931ac9eed294c4fe7c726d8cc2c4d9a39ec12828 100644 --- a/paddle/operators/conv2dtranspose_op.cu +++ b/paddle/operators/conv2d_transpose_op.cu @@ -12,13 +12,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/conv2dtranspose_op.h" +#include "paddle/operators/conv2d_transpose_op.h" namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL( - conv2dtranspose, + conv2d_transpose, ops::GemmConv2DTransposeKernel); REGISTER_OP_GPU_KERNEL( - conv2dtranspose_grad, + conv2d_transpose_grad, ops::GemmConv2DTransposeGradKernel); diff --git a/paddle/operators/conv2dtranspose_op.h b/paddle/operators/conv2d_transpose_op.h similarity index 99% rename from paddle/operators/conv2dtranspose_op.h rename to paddle/operators/conv2d_transpose_op.h index 8c70b3dcec1e26ab3d8a42d88040764c643b5ae6..cab7788227690621a0e5b744197b86c515bbef72 100644 --- a/paddle/operators/conv2dtranspose_op.h +++ b/paddle/operators/conv2d_transpose_op.h @@ -62,7 +62,7 @@ class GemmConv2DTransposeKernel : public framework::OpKernel { std::vector strides = context.Attr>("strides"); // TODO(Zhuoyuan): Paddings can be added in future. - // groups will alway be disabled in conv2dtranspose. + // groups will alway be disabled in conv2d_transpose. const int batch_size = input->dims()[0]; const int m = input->dims()[1]; diff --git a/paddle/operators/conv_cudnn_op.cc b/paddle/operators/conv_cudnn_op.cc index 37bba3a1a1ef0b2e27096498744fccc0162ed4d1..a068daf9a812f6f54084f17ef513d5d96309c182 100644 --- a/paddle/operators/conv_cudnn_op.cc +++ b/paddle/operators/conv_cudnn_op.cc @@ -29,7 +29,7 @@ class CudnnConvOpMaker : public Conv2DOpMaker { "workspace is a section of GPU memory which will be " "allocated/freed each time the operator runs, larger " "workspace size can increase performance but also requires " - "better hardward. This size should be carefully setted.") + "better hardware. This size should be chosen carefully.") .SetDefault(4096); } }; diff --git a/paddle/operators/conv_shift_op.cc b/paddle/operators/conv_shift_op.cc index 6156a2d6af9a010240449a7c944ec0caffc85189..a4150a5664690e750d2501a1849767c23209186b 100644 --- a/paddle/operators/conv_shift_op.cc +++ b/paddle/operators/conv_shift_op.cc @@ -96,14 +96,13 @@ as used in the Neural Turing Machine: https://arxiv.org/abs/1410.5401 The equation is: - \f[ - Out[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} X_{i+j} * Y_{j} - \f] +$$Out[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} X_{i+j} * Y_{j}$$ -where X's index is computed modulo M, and b's index is computed modulo N. +where X's index is computed modulo M, and Y's index is computed modulo N. + +Both inputs X and Y can carry LoD (Level of Details) information. +However, the output only shares the LoD information with input X. -Both of the input `X` and `Y` can carry LoD (Level of Details) information. -However, the output only shares the LoD information with input `X`. )DOC"); } }; diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc index 55f69fb03ad69c94dc4ebb8edd651d84e06a5f46..312264ccd48d1405a247a2c864d9f5897c897bea 100644 --- a/paddle/operators/cos_sim_op.cc +++ b/paddle/operators/cos_sim_op.cc @@ -79,15 +79,16 @@ class CosSimOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Cosine Similarity Operator. -The equation is: Out = X^T * Y / (sqrt(X^T * X) * sqrt(Y^T * Y)). +$Out = X^T * Y / (\sqrt{X^T * X} * \sqrt{Y^T * Y})$ -The input `X` and `Y` must have the same shape, except that the 1st dimension -of input `Y` could be just 1 (different from input `X`), which will be -broadcasted to match the shape of input `X` before computing their cosine +The input X and Y must have the same shape, except that the 1st dimension +of input Y could be just 1 (different from input X), which will be +broadcasted to match the shape of input X before computing their cosine similarity. -Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +Both the input X and Y can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD information with input X. + )DOC"); } }; diff --git a/paddle/operators/crf_decoding_op.cc b/paddle/operators/crf_decoding_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..d1ce74c4b911545476f0b362df0e0f7a0d14cfb4 --- /dev/null +++ b/paddle/operators/crf_decoding_op.cc @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/crf_decoding_op.h" + +namespace paddle { +namespace operators { +class CRFDecodingOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CRFDecodingOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Emission", + "(LoDTensor, default: LoDTensor). A LoDTensor with shape " + "[N x D] where N is the size of the mini-batch and D is the total " + "tag number. This input is the unscaled emission weight matrix of " + "the linear_chain_crf operator."); + AddInput( + "Transition", + "(Tensor, default: Tensor). A Tensor with shape [(D + 2) x D]. " + "This input is the transition weights learned by the linear_chain_crf " + "operator, denoted as w. The 1st row of w are transition weights for " + "the start mask. The 2nd row of w are transition weights for the end " + "mask. Transition weights between other tags begin from the 3rd row of " + "w. See more details in comments of the linear_chain_crf operator."); + AddInput( + "Label", + "(LoDTensor, LoDTensor). The ground truth with shape " + "[N x 1]. This input is optional. See more details in the operator's " + "comments.") + .AsDispensable(); + AddOutput("ViterbiPath", + "(LoDTensor, LoDTensor). The decoding results. What to " + "return changes depending on whether the Input(Label) (the groud " + "truth) is given. See more details in the operator's comment."); + AddComment(R"DOC( +The crf_decoding operator reads the emission feature weights and the transition +freature weights learned by the linear_chain_crf operator. It implements the +Viterbi algorithm which is a dynamic programming algorithm for finding the most +likely sequence of hidden states, called the Viterbi path, that results in a +sequence of observed tags. + +The output of this operator changes according to whether Input(Label) is given: + +1. Input(Label) is given: + +This happens in training. This operator is used to co-work with the chunk_eval +operator. + +When Input(Label) is given, the crf_decoding operator returns a row vector +with shape [N x 1] whose values are fixed to be 0, indicating an incorrect +prediction, or 1 indicating a tag is correctly predicted. Such an ouput is the +input to chunk_eval operator. + +2. Input(Label) is not given: + +This is the standard decoding process. + +The crf_decoding operator returns a row vecotr with shape [N x 1] whose values +range from 0 to maximum tag number - 1. Each element indicates an index of a +predicted tag. +)DOC"); + } +}; + +class CRFDecodingOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Emission"), + "Input(Emission) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Transition"), + "Input(Transition) should be not null."); + + PADDLE_ENFORCE(ctx->HasOutput("ViterbiPath"), + "Output(ViterbiPath) should be not null."); + + auto emission_dims = ctx->GetInputDim("Emission"); + PADDLE_ENFORCE_EQ(emission_dims.size(), 2UL, + "The Input(Emission) should be a 2-D tensor."); + PADDLE_ENFORCE(emission_dims[0], "An empty mini-batch is not allowed."); + + auto transition_dims = ctx->GetInputDim("Transition"); + PADDLE_ENFORCE_EQ(transition_dims.size(), 2UL, + "The Input(Transition) should be a 2-D tensor."); + PADDLE_ENFORCE_EQ( + transition_dims[0] - 2, transition_dims[1], + "An invalid dimension for the Input(Transition), which should " + "be a 2-D tensor with shape [(D + 2) x D]."); + PADDLE_ENFORCE_EQ( + emission_dims[1], transition_dims[1], + "The 2nd dimension of the Input(Emission) and the Input(Transition) " + "should be equal to the tag number."); + + if (ctx->HasInput("Label")) { + auto label_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE(label_dims.size() == 2UL && label_dims[1] == 1UL, + "The Input(Label) should be a 2-D tensor with the 2nd " + "dimensions fixed to 1."); + PADDLE_ENFORCE_EQ( + emission_dims[0], label_dims[0], + "The height of Input(Emission) and the height of Input(Label) " + "should be the same."); + } + + ctx->ShareLoD("Emission", /*->*/ "ViterbiPath"); + ctx->SetOutputDim("ViterbiPath", {emission_dims[0], 1}); + } + + protected: + framework::DataType IndicateDataType( + const framework::ExecutionContext& ctx) const override { + return framework::ToDataType(ctx.Input("Emission")->type()); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(crf_decoding, ops::CRFDecodingOp, + ops::CRFDecodingOpMaker); +REGISTER_OP_CPU_KERNEL( + crf_decoding, ops::CRFDecodingOpKernel, + ops::CRFDecodingOpKernel); diff --git a/paddle/operators/crf_decoding_op.h b/paddle/operators/crf_decoding_op.h new file mode 100644 index 0000000000000000000000000000000000000000..526e0c5dcb2649b35ee28f5153c8472ca7a0af7b --- /dev/null +++ b/paddle/operators/crf_decoding_op.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using framework::LoDTensor; +using framework::LoD; +using framework::Tensor; + +template +class CRFDecodingOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "The crf_decoding operator can only run on CPU."); + + auto* emission_weights = ctx.Input("Emission"); + auto* transition_weights = ctx.Input("Transition"); + auto* label = ctx.Input("Label"); + auto* decoded_path = ctx.Output("ViterbiPath"); + + PADDLE_ENFORCE_EQ(emission_weights->NumLevels(), 1UL, + "The Input(Emission) should be a sequence."); + auto lod = emission_weights->lod(); + PADDLE_ENFORCE(lod.size(), "Input(Emission) must be a sequence."); + const size_t level = 0; + const size_t seq_num = lod[level].size() - 1; + + int* path = decoded_path->mutable_data(platform::CPUPlace()); + math::SetConstant()(ctx.device_context(), + decoded_path, 0); + for (size_t i = 0; i < seq_num; ++i) { + int start_pos = static_cast(lod[level][i]); + int end_pos = static_cast(lod[level][i + 1]); + Tensor decoded_path_one_seq = decoded_path->Slice(start_pos, end_pos); + Decode(emission_weights->Slice(start_pos, end_pos), *transition_weights, + &decoded_path_one_seq); + } + + if (label) { + PADDLE_ENFORCE_EQ(label->NumLevels(), 1UL, + "The Input(Label) should be a sequence."); + const int* label_value = label->data(); + size_t batch_size = emission_weights->dims()[0]; + for (size_t i = 0; i < batch_size; ++i) { + path[i] = label_value[i] == path[i] ? 1 : 0; + } + } + } + + private: + void Decode(const Tensor& emission_weights, const Tensor& transition_weights, + Tensor* decoded_path) const { + auto emission_dims = emission_weights.dims(); + const size_t seq_len = emission_dims[0]; + const size_t tag_num = emission_dims[1]; + + const size_t state_trans_base_idx = 2; + + const T* x = emission_weights.data(); + const T* w = transition_weights.data(); + int* path = decoded_path->data(); + + // alpha is a memo table. An element alpha(k, v) records the score of the + // best sequence of tags from position 1 to position k with v being the end + // tag. + Tensor alpha; + T* alpha_value = alpha.mutable_data(emission_dims, platform::CPUPlace()); + Tensor track; + int* track_value = + track.mutable_data(emission_dims, platform::CPUPlace()); + + for (size_t i = 0; i < tag_num; ++i) alpha_value[i] = w[i] + x[i]; + + for (size_t k = 1; k < seq_len; ++k) { + for (size_t i = 0; i < tag_num; ++i) { + T max_score = -std::numeric_limits::max(); + int max_j = 0; + for (size_t j = 0; j < tag_num; ++j) { + T score = alpha_value[(k - 1) * tag_num + j] + + w[(j + state_trans_base_idx) * tag_num + i]; + if (score > max_score) { + max_score = score; + max_j = j; + } + } + + alpha_value[k * tag_num + i] = max_score + x[k * tag_num + i]; + track_value[k * tag_num + i] = max_j; + } + } + + T max_score = -std::numeric_limits::max(); + int max_i = 0; + for (size_t i = 0; i < tag_num; ++i) { + T score = alpha_value[(seq_len - 1) * tag_num + i] + w[tag_num + i]; + if (score > max_score) { + max_score = score; + max_i = i; + } + } + path[seq_len - 1] = max_i; + for (int k = seq_len - 1; k >= 1; --k) { + path[k - 1] = max_i = track_value[k * tag_num + max_i]; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc index ed78e9e3a3a49b7ff0990b8d13cfe2dae594b722..6752eb8c1c72150b0b1cf5595211ca1d01ef2bf4 100644 --- a/paddle/operators/crop_op.cc +++ b/paddle/operators/crop_op.cc @@ -56,34 +56,35 @@ class CropOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of pad op. " - "The input should be a k-D tensor(k > 0 and k < 7)"); + "The input should be a k-D tensor(k > 0 and k < 7)."); AddInput("Y", - "The input used as reference for cropping" - " with the same dimension as X. ") + "The input used as reference for cropping, " + "which is of the same dimensions as X.") .AsDispensable(); AddOutput("Out", - "The output of crop op " - "with the same dimension as X."); + "The output of crop op, " + "which is of the same dimensions as X."); AddAttr>("offsets", - "A list describing offsets to be cropped." - "The size of offsets list should be as same as " - "dimension size of input X."); + "A list describing offsets to be cropped. " + "The size of offsets list should be the same as " + "the dimension size of input X."); AddAttr>("shape", - "A list describing the shape of output." - "The size of shape list should be as same as " - "dimension size of input X.") + "A list describing the shape of output. " + "The size of shape list should be the same as " + "the dimension size of input X.") .SetDefault(std::vector()); AddComment(R"DOC( Crop Operator. + Crop input into output, as specified by offsets and shape. There are two ways to set shape: -1. referenc input: crop input X as shape as reference input. +1. reference input: crop input X into the same shape as reference input. The dimension of reference input should - be as same as input X. -2. shape list: crop input X by shape described by a list. - The size of shape list should be as same as - dimension size of input X. + be the same as the dimension of input X. +2. shape list: crop input X into the shape described by a list. + The size of shape list should be the same as + the dimension size of input X. The input should be a k-D tensor(k > 0 and k < 7). As an example: @@ -91,20 +92,20 @@ Given: X = [[0, 1, 2, 0, 0] [0, 3, 4, 0, 0] - [0, 0, 0, 0, 0]] + [0, 0, 0, 0, 0]], and - offsets = [0, 1] + offsets = [0, 1], and - shape = [2, 2] + shape = [2, 2], -then we get +we get: Out = [[1, 2], - [3, 4]] + [3, 4]]. )DOC"); } diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index d94b96200c2a5cd112b17e45aa6cd4a63bdd04d0..24df1fcadac75315890635f4d3aaa7146c1cc27b 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -28,8 +28,9 @@ class CrossEntropyOp : public framework::OperatorWithKernel { auto x_dims = ctx->GetInputDim("X"); auto label_dims = ctx->GetInputDim("Label"); - PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank should be 2."); - PADDLE_ENFORCE_EQ(label_dims.size(), 2, "Input(Label)'s rank should be 2."); + PADDLE_ENFORCE_EQ(x_dims.size(), 2UL, "Input(X)'s rank should be 2."); + PADDLE_ENFORCE_EQ(label_dims.size(), 2UL, + "Input(Label)'s rank should be 2."); PADDLE_ENFORCE_EQ(x_dims[0], label_dims[0], "The 1st dimension of Input(X) and Input(Label) should " "be equal."); @@ -38,8 +39,8 @@ class CrossEntropyOp : public framework::OperatorWithKernel { "If Attr(soft_label) == true, the 2nd dimension of " "Input(X) and Input(Label) should be equal."); } else { - PADDLE_ENFORCE_EQ(label_dims[1], 1, - "If Attr(soft_label) == false, the 2nd dimension of " + PADDLE_ENFORCE_EQ(label_dims[1], 1UL, + "If Attr(softLabel) == false, the 2nd dimension of " "Input(Label) should be 1."); } @@ -48,7 +49,8 @@ class CrossEntropyOp : public framework::OperatorWithKernel { } protected: - // CrossEntropy's data type just determined by "X" + // Explicitly set that the data type of computation kernel of cross_entropy + // is determined by its input "X". framework::DataType IndicateDataType( const framework::ExecutionContext& ctx) const override { return framework::ToDataType(ctx.Input("X")->type()); @@ -94,7 +96,8 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel { } protected: - // CrossEntropy's data type just determined by "X" + // Explicitly set that the data type of computation kernel of cross_entropy + // is determined by its input "X". framework::DataType IndicateDataType( const framework::ExecutionContext& ctx) const override { return framework::ToDataType(ctx.Input("X")->type()); @@ -115,9 +118,9 @@ class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { "Label", "(Tensor, default Tensor), the ground truth which is " "a 2-D tensor. " - "When soft_label is set to false, `Label` is a Tensor with shape " + "When soft_label is set to false, Label is a Tensor with shape " "[N x 1]. " - "When soft_label is set to true, `Label` is a Tensor " + "When soft_label is set to true, Label is a Tensor " "with shape [N x K]."); AddOutput("Y", "(Tensor, default Tensor), a 2-D tensor " @@ -135,13 +138,13 @@ computation. 1) One-hot cross-entropy: soft_label = false, Label[i, 0] indicates the class index for sample i: - Y[i] = -log(X[i, Label[i]]) + $Y[i] = -\log(X[i, Label[i]])$ 2) Soft-label cross-entropy: soft_label = true, Label[i, j] indicates the soft label of class j for sample i: - Y[i] = \sum_j{-Label[i, j] * log(X[i, j])} + $Y[i] = \sum_j{-Label[i, j] * log(X[i, j])}$ Please make sure that in this case the summuation of each row of Label equals one. @@ -151,8 +154,9 @@ computation. non-zero element (equals 1), soft-label cross-entropy degenerates to a one-hot cross-entropy with one-hot label representation. -Both the input `X` and `Label` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +Both the input X and Label can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD information with input X. + )DOC"); } }; diff --git a/paddle/operators/decayed_adagrad_op.cc b/paddle/operators/decayed_adagrad_op.cc index 17b394aa07cb0c7ca6e085b61590ff052221b22c..640b4e77448d1b64bcf7375f26c07ff1d2bdeaa3 100644 --- a/paddle/operators/decayed_adagrad_op.cc +++ b/paddle/operators/decayed_adagrad_op.cc @@ -75,11 +75,18 @@ class DecayedAdagradOpMaker : public framework::OpProtoAndCheckerMaker { "Constant for numerical stability") .SetDefault(1.0e-6f); AddComment(R"DOC( +Decayed Adagrad Optimizer. -Decayed Adagrad +The update is done as follows: -moment_out = decay * moment + (1 - decay) * grad * grad -param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon) +$$ +moment\_out = decay * moment + (1 - decay) * grad * grad \\ +param\_out = param - \frac{learning\_rate * grad}{\sqrt{moment\_out} + epsilon} +$$ + +The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) +does not have an epsilon attribute. It is added here for numerical +stability to avoid the division by zero error. )DOC"); } diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc index ff1ccea3b94dcd55c372b707c2afeda874ed212e..818146aca766cb13b93fd024c11c1209655d9e11 100644 --- a/paddle/operators/dropout_op.cc +++ b/paddle/operators/dropout_op.cc @@ -43,22 +43,24 @@ class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { DropoutOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddAttr("dropout_prob", "Probability of setting units to zero.") - .SetDefault(.5f); - AddAttr("is_training", "Whether in training phase.").SetDefault(true); - AddAttr("seed", "Dropout random seed.").SetDefault(0); AddInput("X", "The input of dropout op."); AddOutput("Out", "The output of dropout op."); AddOutput("Mask", "The random sampled dropout mask.").AsIntermediate(); + AddAttr("dropout_prob", "Probability of setting units to zero.") + .SetDefault(.5f); + AddAttr("is_training", "True if in training phase.").SetDefault(true); + AddAttr("seed", "Dropout random seed.").SetDefault(0); + AddComment(R"DOC( Dropout Operator. -'Dropout' refers to randomly dropping out units in a nerual network. It is a +Dropout refers to randomly dropping out units in a nerual network. It is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly set (according to the given dropout probability) the outputs of some units to zero, while others -being set to their inputs. +are set equal to their corresponding inputs. + )DOC"); } }; diff --git a/paddle/operators/dynamic_recurrent_op.cc b/paddle/operators/dynamic_recurrent_op.cc index a0b06ac1dc305bc899f9abaafcc980a6150ecda9..d48cc4e8df587708ab93e7d788145adc01c1d3e5 100644 --- a/paddle/operators/dynamic_recurrent_op.cc +++ b/paddle/operators/dynamic_recurrent_op.cc @@ -386,12 +386,13 @@ class DynamicRecurrentOpProtoAndCheckerMaker RNNAlgorithm::kArgNames[RNNAlgorithm::ComputeMode::kForward]; // inputs and outputs stored in proto AddInput(name.inlinks, - "the inputs that need to be segmented for each step.") + "The inputs that need to be segmented for each step.") .AsDuplicable(); - AddInput(name.initial_states, "variables to initialize states.") + AddInput(name.initial_states, "Variables to initialize the states.") .AsDuplicable(); - AddOutput(name.outlinks, "the outputs that need to concated for all steps.") + AddOutput(name.outlinks, + "The outputs that need to be concatenated for all steps.") .AsDuplicable(); AddOutput(name.step_scopes, "step scopes"); @@ -399,7 +400,12 @@ class DynamicRecurrentOpProtoAndCheckerMaker AddAttr>(name.ex_states, "names of ex_states"); AddAttr>(name.states, "names of states"); - AddComment("This is a RNN operator for varience-length sequences."); + AddComment(R"DOC( +Dynamic Recurrent Operator. + +This is a RNN operator for varience-length sequences. + +)DOC"); } }; diff --git a/paddle/operators/dynamic_recurrent_op_test.cc b/paddle/operators/dynamic_recurrent_op_test.cc index fff63efb24c70b7e864e2d5b011a22883c13dede..8d840e259b190ead86a66df8ab31c5170db4d824 100644 --- a/paddle/operators/dynamic_recurrent_op_test.cc +++ b/paddle/operators/dynamic_recurrent_op_test.cc @@ -51,7 +51,7 @@ class RNNAlgorithmTestHelper : public ::testing::Test { CreateGlobalVariables(); auto op_desc = CreateOpDesc(); - op = paddle::framework::OpRegistry::CreateOp(op_desc, nullptr); + op = paddle::framework::OpRegistry::CreateOp(op_desc); dop = &(dynamic_cast(op.get())->rnn); InitCacheManually(); InitStepNet(); diff --git a/paddle/operators/elementwise_add_op.cc b/paddle/operators/elementwise_add_op.cc index d9bc80c869c023caebf0b45ed24f2def3f0b1dd8..ebe1de90c7d245756de759d8675a30f955843798 100644 --- a/paddle/operators/elementwise_add_op.cc +++ b/paddle/operators/elementwise_add_op.cc @@ -22,7 +22,7 @@ class ElementwiseAddOpMaker : public ElementwiseOpMaker { ElementwiseAddOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("add", "Out = X + Y"); + SetComment("Add", "$Out = X + Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_div_op.cc b/paddle/operators/elementwise_div_op.cc index 3f56344d0007b5f14fd9b5b9b44a9b29d3c42f2a..de75816a249002549940b04d928c88c17d075917 100644 --- a/paddle/operators/elementwise_div_op.cc +++ b/paddle/operators/elementwise_div_op.cc @@ -22,7 +22,7 @@ class ElementwiseDivOpMaker : public ElementwiseOpMaker { ElementwiseDivOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Div", "Out = X / Y"); + SetComment("Div", "$Out = X / Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc index da7765aa6a7a81c9e0b4f462022cad54c16aec47..ffa10486f123963274aa478eb4c607e32138bcec 100644 --- a/paddle/operators/elementwise_mul_op.cc +++ b/paddle/operators/elementwise_mul_op.cc @@ -23,7 +23,7 @@ class ElementwiseMulOpMaker : public ElementwiseOpMaker { ElementwiseMulOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Mul", "Out = X ⊙ Y"); + SetComment("Mul", "$Out = X \\odot\\ Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h index fce4b24a22f40c9cc57738273a758d0d48ff5e91..56e5eb69bc382a2c15d88b759fa6987f02c6cabb 100644 --- a/paddle/operators/elementwise_op.h +++ b/paddle/operators/elementwise_op.h @@ -46,37 +46,42 @@ class ElementwiseOpMaker : public framework::OpProtoAndCheckerMaker { ElementwiseOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", R"DOC( -The first input of elementwise op, it's a tensor of any dimensions. -)DOC"); - AddInput("Y", R"DOC( -The sencond input of elementwise op, it's a tensor and it's dimensions -must be small or equal to X's dimensions. -)DOC"); + AddInput("X", "(Tensor) The first input tensor of elementwise op"); + AddInput("Y", "(Tensor) The second input tensor of elementwise op"); + AddOutput("Out", "The output of elementwise op"); AddAttr("axis", - R"DOC( -When the shape(Y) does not equal the shape(X),Y will be broadcasted -to match the shape of X and axis should be dimension index Y in X - )DOC") + "(int, default -1) The starting dimension index " + "for broadcasting Y onto X") .SetDefault(-1) .EqualGreaterThan(-1); - - AddOutput("Out", "The output of elementwise op"); comment_ = R"DOC( -Limited elementwise {name} operator.The equation is: Out = {equation}. -1. The shape of Y should be same with X or -2. Y's shape is a subset of X. - Y will be broadcasted to match the shape of X and axis should be dimension index Y in X. - - example: - shape(X) = (2, 3, 4, 5), shape(Y) = (,) - shape(X) = (2, 3, 4, 5), shape(Y) = (5,) - shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) - shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 - shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 +Limited Elementwise {name} Operator. + +The equation is: + +{equation} + +X is a tensor of any dimension and the dimensions of tensor Y must be smaller than +or equal to the dimensions of X. + +There are two cases for this operator: +1. The shape of Y is same with X; +2. The shape of Y is a subset of X. + +For case 2: +Y will be broadcasted to match the shape of X and axis should be +the starting dimension index for broadcasting Y onto X. + +example: + shape(X) = (2, 3, 4, 5), shape(Y) = (,) + shape(X) = (2, 3, 4, 5), shape(Y) = (5,) + shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) + shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 + shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 Both the input X and Y can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input X. +or not. But the output only shares the LoD information with input X. + )DOC"; AddComment(comment_); } diff --git a/paddle/operators/elementwise_sub_op.cc b/paddle/operators/elementwise_sub_op.cc index 3e4f98fdb35b148931a67d511fe41958eb523f99..39702dad0ee61de71ff0d54765e6f73de93cee9c 100644 --- a/paddle/operators/elementwise_sub_op.cc +++ b/paddle/operators/elementwise_sub_op.cc @@ -22,7 +22,7 @@ class ElementwiseSubOpMaker : public ElementwiseOpMaker { ElementwiseSubOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Sub", "Out = X - Y"); + SetComment("Sub", "$Out = X - Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/feed_op.cc b/paddle/operators/feed_op.cc index 0e5b263eae904d97b61d41691b848e4fa2c17971..0dd84cbeaafbafd45132b0a0b744554ce7475411 100644 --- a/paddle/operators/feed_op.cc +++ b/paddle/operators/feed_op.cc @@ -59,8 +59,13 @@ class FeedOpInfoMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of feed op"); AddOutput("Out", "The output of feed op"); - AddComment("feed op, it should not be configured by users directly"); - AddAttr("col", "column of feed"); + AddAttr("col", "(int) The column of feed"); + AddComment(R"DOC( +Feed Operator. + +It should not be configured by users directly. + +)DOC"); } }; diff --git a/paddle/operators/fetch_op.cc b/paddle/operators/fetch_op.cc index f1086e3dc774a5e57f1abb5d4f91f859fc0e64aa..8108ae69dec4bafd1c04d5ab05eef6f467d4c6e8 100644 --- a/paddle/operators/fetch_op.cc +++ b/paddle/operators/fetch_op.cc @@ -66,8 +66,13 @@ class FetchOpInfoMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of fetch op"); AddOutput("Out", "The output of fetch op"); - AddComment("fetch op, it should not be configured by users directly"); - AddAttr("col", "column of fetch"); + AddAttr("col", "(int) The column of fetch"); + AddComment(R"DOC( +Fetch Operator. + +It should not be configured by users directly. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/fill_constant_batch_size_like_op.cc b/paddle/operators/fill_constant_batch_size_like_op.cc index 58c9f1cd2c79c150aaed7753641f6ad6120dd0f5..3f02214f30f886c5a03ad438fd795d44a0b8dddc 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cc +++ b/paddle/operators/fill_constant_batch_size_like_op.cc @@ -36,7 +36,12 @@ class FillConstantBatchSizeLikeOp : public framework::OperatorWithKernel { [](int a) { return static_cast(a); }); auto dims = framework::make_ddim(shape_int64); - dims[0] = ctx->GetInputDim("Input")[0]; + int dim_idx = ctx->Attrs().Get("dim_idx"); + PADDLE_ENFORCE_GE(dim_idx, 0); + PADDLE_ENFORCE_GT(static_cast(shape.size()), dim_idx); + PADDLE_ENFORCE_GT(ctx->GetInputDim("Input").size(), dim_idx); + + dims[dim_idx] = ctx->GetInputDim("Input")[dim_idx]; ctx->SetOutputDim("Out", dims); } @@ -57,16 +62,24 @@ class FillConstantBatchSizeLikeOpMaker "(int, default 5 (FP32)) " "Output data type") .SetDefault(framework::DataType::FP32); - AddAttr>("shape", "(vector) The shape of the output"); - AddAttr("value", "(float, default 0) The value to be filled") - .SetDefault(0.0f); AddInput("Input", "(Tensor) Tensor " - "whose first dimension is used to specify the batch_size"); + "whose dim_idx th dimension is used to specify the batch_size"); AddOutput("Out", "(Tensor) Tensor of specified shape will be filled " "with the specified value"); - AddComment(R"DOC(Fill up a variable with specified constant value.)DOC"); + AddAttr>("shape", "(vector) The shape of the output"); + AddAttr("dim_idx", + "(int, default 0) The index of batch size dimension") + .SetDefault(0); + AddAttr("value", "(float, default 0) The value to be filled") + .SetDefault(0.0f); + AddComment(R"DOC( +FillConstantBatchSizeLike Operator. + +Fill up a variable with specified constant value. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/fill_constant_op.cc b/paddle/operators/fill_constant_op.cc index 7a861b6cfc0fab312f4e5a7cce2fc28f923173d2..ee2219cd03313beae352c18adbe7e029fabbb6d3 100644 --- a/paddle/operators/fill_constant_op.cc +++ b/paddle/operators/fill_constant_op.cc @@ -54,7 +54,12 @@ class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor) Tensor of specified shape will be filled " "with the specified value"); - AddComment(R"DOC(Fill up a variable with specified constant value.)DOC"); + AddComment(R"DOC( +FillConstantBatchSizeLike Operator. + +Fill up a variable with specified constant value. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index ed529ac40aaf179b35a9ab32e11ed7dbbe9289ba..8ab39d4fb012b8fa3883f33e4d15be7918500354 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -37,11 +37,13 @@ class FillZerosLikeOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of fill-zeros-like op."); - AddOutput("Y", "The varibale will be filled up with zeros."); + AddOutput("Y", "The variable will be filled up with zeros."); AddComment(R"DOC( -Fill up a vriable with zeros. +FillZerosLike Operator. + +Fill up a variable with zeros. +The output will have the same size as the input. -The output will have the same size with input. )DOC"); } }; diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc index f6c7f472da24a1a60c0d2538ae643bdc8e55b10f..aee672500ee5d1bf6cc7ef872f2cb6c408de6d9e 100644 --- a/paddle/operators/gather_op.cc +++ b/paddle/operators/gather_op.cc @@ -67,11 +67,28 @@ class GatherOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The source input of gather op"); AddInput("Index", "The index input of gather op"); - AddOutput("Out", "The output of add op"); + AddOutput("Out", "The output of gather op"); AddComment(R"DOC( -Gather Operator by selecting from the first axis, +Gather Operator. + +$Out = X[Index]$ + +Out is obtained by gathering entries of the outer-most dimension +of X indexed by Index and concatenate them together. + +Example: + +X = [[1, 2], + [3, 4], + [5, 6]] + +Index = [[1, 2]] + +Then: + +Out = [[3, 4], + [5, 6]] -Out = X[Index] )DOC"); } }; diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index 04dfdf7c48381240108cf924979764966599151f..802c98ae764d02af3143d1d39b714d486791da82 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -45,14 +45,14 @@ class GaussianRandomOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of GaussianRandomOp should not be null."); - auto dims = ctx->Attrs().Get>("dims"); + auto shape = ctx->Attrs().Get>("shape"); std::vector temp; - temp.reserve(dims.size()); - for (auto dim : dims) { + temp.reserve(shape.size()); + for (auto dim : shape) { temp.push_back(static_cast(dim)); } - PADDLE_ENFORCE(dims.size() > 0UL, - "dims can be one int or array. dims must be set."); + PADDLE_ENFORCE(shape.size() > 0UL, + "shape can be one int or array. shape must be set."); ctx->SetOutputDim("Out", framework::make_ddim(temp)); } @@ -68,21 +68,35 @@ class GaussianRandomOpMaker : public framework::OpProtoAndCheckerMaker { GaussianRandomOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "output matrix of random op"); - AddComment(R"DOC( -GaussianRandom operator. -Use to initialize tensor with gaussian random generator. -)DOC"); + AddOutput("Out", "Output matrix of gaussian random op"); - AddAttr>("dims", "The dimension of random tensor."); - AddAttr("mean", "mean of random tensor.").SetDefault(.0f); - AddAttr("std", "std of random tensor.").SetDefault(1.0f); + AddAttr>("shape", + "(vector) " + "The dimension of random tensor."); + AddAttr("mean", + "(float, default 0.0) " + "mean of random tensor.") + .SetDefault(.0f); + AddAttr("std", + "(float, default 1.0) " + "std of random tensor.") + .SetDefault(1.0f); AddAttr("seed", + "(int, default 0) " "Random seed of generator." - "0 means use system wide seed") + "0 means use system wide seed.") .SetDefault(0); - AddAttr("data_type", "output data type") + AddAttr("data_type", + "(int, default 5(FP32)) " + "Output data type.") .SetDefault(framework::DataType::FP32); + + AddComment(R"DOC( +GaussianRandom Operator. + +Used to initialize tensors with gaussian random generator. + +)DOC"); } }; diff --git a/paddle/operators/gru_unit_op.cc b/paddle/operators/gru_unit_op.cc index 8d9723289d9af9ef218a5e056b4b585383e00dac..89c027ff1eea93012dc5ab22b081786efc328e96 100644 --- a/paddle/operators/gru_unit_op.cc +++ b/paddle/operators/gru_unit_op.cc @@ -80,19 +80,21 @@ class GRUUnitOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("HiddenPrev", "(Tensor) Matrix with shape [batch_size, frame_size] for the " "states of previous time step."); - AddInput("Weight", - "(Tensor) Weight matrix with shape [frame_size, frame_size * 3]. " - "The elements continuous in memory can be divided into two parts. " - "The first part are weights of the update gate and reset gate " - "with shape [frame_size, frame_size * 2], and the second part are " - "weights of output candidate with shape [frame_size, frame_size]"); - AddInput("Bias", - "(Tensor) Bias vector with shape [1, frame_size * 3] concating " - "bias of the update gate, reset gate and output candidate.") + AddInput( + "Weight", + "(Tensor) Weight matrix with shape [frame_size, frame_size * 3]. " + "The elements continuous in memory can be divided into two parts. " + "The first part are weights of the update gate and reset gate " + "with shape [frame_size, frame_size * 2], and the second part are " + "weights of output candidate with shape [frame_size, frame_size]."); + AddInput( + "Bias", + "(Tensor) Bias vector with shape [1, frame_size * 3] concatenating " + "bias of the update gate, reset gate and output candidate.") .AsDispensable(); AddOutput("Gate", "(Tensor) Matrix with shape [batch_size, frame_size * 3] for the " - "output of update gate, reset gate and output candidate") + "output of update gate, reset gate and output candidate.") .AsIntermediate(); AddOutput("ResetHiddenPrev", "(Tensor) Matrix with shape [batch_size, frame_size] for the " @@ -112,16 +114,19 @@ class GRUUnitOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(sigmoid) .InEnum({identity, sigmoid, tanh, relu}); AddComment(R"DOC( -GRUUnitOp implements part calculations of the GRU unit as following: +GRUUnit Operator. -\f[ -update \ gate: u_t = actGate(xu_t + W_u * hidden_prev + bias_u) \\ -reset \ gate: r_t = actGate(xr_t + W_r * hidden_prev + bias_r) \\ -output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, hidden_prev) + bias_c) \\ -output: h_t = dot((1-u_t), {h}_t) + dot(u_t, hidden_prev) -\f] +This operator implements partial calculations of the GRU unit as follows: + +$$ +update \ gate: u_t = actGate(xu_t + W_u * hidden_{prev} + bias_u) \\ +reset \ gate: r_t = actGate(xr_t + W_r * hidden_{prev} + bias_r) \\ +output \ candidate: {h}_t = actNode({xc}_t + W_c * dot(r_t, hidden_{prev}) + bias_c) \\ +output: h_t = dot((1-u_t), {h}_t) + dot(u_t, hidden_{prev}) +$$ The rest of GRU unit can be completed by using FCOp's output as the input of GRUUnitOp. + )DOC"); } }; diff --git a/paddle/operators/huber_loss_op.cc b/paddle/operators/huber_loss_op.cc index 2d9449f5ca50dab8d2a7928c4311ec2d66b47904..3435e74b0afb470fcbd1c0f4e06ad363352cac00 100644 --- a/paddle/operators/huber_loss_op.cc +++ b/paddle/operators/huber_loss_op.cc @@ -59,10 +59,12 @@ class HuberLossOpMaker : public framework::OpProtoAndCheckerMaker { "The shape is same as Input(X) and will be reused in backward.") .AsIntermediate(); AddOutput("Out", - "The output tensor with shape [batch_size, 1] which represents " - "the huber loss."); + "The output tensor with shape [batch_size, 1] " + "which represents the huber loss."); AddAttr("delta", "Hyper parameter in huber loss."); AddComment(R"DOC( +HuberLoss Operator. + Huber loss is a loss function used in robust regression. We define X as the input value and Y as the target value. Huber loss can evaluate the fitness of X to Y. Different from MSE loss, Huber loss is more robust for outliers. The diff --git a/paddle/operators/increment_op.cc b/paddle/operators/increment_op.cc index 139392c691e00b2a94f46801f1cfc2018ce139f5..c3e9308fe0ad6a90ce5e9097d078dfe3a3e1c20c 100644 --- a/paddle/operators/increment_op.cc +++ b/paddle/operators/increment_op.cc @@ -39,14 +39,18 @@ class IncrementOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) The input tensor of increment operator"); AddOutput("Out", "(Tensor) The output tensor of increment operator."); - AddComment(R"DOC(Increment operator - -The equation is: Out = X + step -)DOC"); AddAttr("step", + "(float, default 1.0) " "The step size by which the " "input tensor will be incremented.") .SetDefault(1.0); + AddComment(R"DOC( +Increment Operator. + +The equation is: +$$Out = X + step$$ + +)DOC"); } }; diff --git a/paddle/operators/l1_norm_op.cc b/paddle/operators/l1_norm_op.cc index 1d111696cf43d232413a8dec7ffb057cb1913c7f..02ebf022968e95d0b20598d3c935fb51177c8841 100644 --- a/paddle/operators/l1_norm_op.cc +++ b/paddle/operators/l1_norm_op.cc @@ -57,7 +57,7 @@ L1 Norm Operator. Computes the L1 norm of a tensor. -Out = sum (abs(X)) +$$Out = \sum{|X|}$$ )DOC"); } diff --git a/paddle/operators/linear_chain_crf_op.cc b/paddle/operators/linear_chain_crf_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..bcb48e13bd948b4e91ce8cbd7231a9619fac8d18 --- /dev/null +++ b/paddle/operators/linear_chain_crf_op.cc @@ -0,0 +1,263 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/linear_chain_crf_op.h" + +namespace paddle { +namespace operators { + +class LinearChainCRFOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LinearChainCRFOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Emission", + "(LoDTensor, default LoDTensor) " + "A 2-D LoDTensor with shape [N x D], where N is the size of the " + "mini-batch and D is the total tag number. The unscaled emission " + "weight matrix for the linear chain CRF. "); + AddInput("Transition", + "(Tensor, default Tensor) A 2-D Tensor with shape " + "[(D + 2) x D]. The learnable parameter for the linear_chain_crf " + "operator. See more details in the operator's comments."); + AddInput("Label", + "(LoDTensor, default LoDTensor) A LoDTensor with shape " + "[N x 1], where N is the total element number in a mini-batch. " + "The ground truth."); + AddOutput( + "Alpha", + "(Tensor, default Tensor) A 2-D Tensor with shape [N x D]. " + "The forward vectors for the entire batch. Denote it as \f$\alpha\f$. " + "\f$\alpha$\f is a memo table used to calculate the normalization " + "factor in CRF. \f$\alpha[k, v]$\f stores the unnormalized " + "probabilites of all possible unfinished sequences of tags that end at " + "position \f$k$\f with tag \f$v$\f. For each \f$k$\f, " + "\f$\alpha[k, v]$\f is a vector of length \f$D$\f with a component for " + "each tag value \f$v$\f. This vector is called a forward vecotr and " + "will also be used in backward computations.") + .AsIntermediate(); + AddOutput( + "EmissionExps", + "(Tensor, default Tensor) A 2-D Tensor with shape [N x D]. " + "The exponentials of Input(Emission). This is an intermediate " + "computational result in forward computation, and will be reused in " + "backward computation.") + .AsIntermediate(); + AddOutput( + "TransitionExps", + "(Tensor, default Tensor) A 2-D Tensor with shape " + "[(D + 2) x D]. The exponentials of Input(Transition). This is an " + "intermediate computational result in forward computation, and " + "will be reused in backward computation.") + .AsIntermediate(); + AddOutput( + "LogLikelihood", + "(Tensor, default Tensor) The logarithm of the conditional " + "likelihood of each training sample in a mini-batch. This is a 2-D " + "tensor with shape [S x 1], where S is the sequence number in a " + "mini-batch. Note: S is equal to the sequence number in a mini-batch. " + "The output is no longer a LoDTensor."); + AddComment(R"DOC( +LinearChainCRF Operator. + +Conditional Random Field defines an undirected probabilistic graph with nodes +denoting random variables and edges denoting dependencies between these +variables. CRF learns the conditional probability \f$P(Y|X)\f$, where +\f$X = (x_1, x_2, ... , x_n)\f$ are structured inputs and +\f$Y = (y_1, y_2, ... , y_n)\f$ are labels for the inputs. + +Linear chain CRF is a special case of CRF that is useful for sequence labeling +task. Sequence labeling tasks do not assume a lot of conditional +independences among inputs. The only constraint they impose is that the input +and output must be linear sequences. Thus, the graph of such a CRF is a simple +chain or a line, which results in the linear chain CRF. + +This operator implements the Forward-Backward algorithm for the linear chain +CRF. Please refer to http://www.cs.columbia.edu/~mcollins/fb.pdf and +http://cseweb.ucsd.edu/~elkan/250Bwinter2012/loglinearCRFs.pdf for details. + +Equation: +1. Denote Input(Emission) to this operator as \f$x\f$ here. +2. The first D values of Input(Transition) to this operator are for starting +weights, denoted as \f$a\f$ here. +3. The next D values of Input(Transition) of this operator are for ending +weights, denoted as \f$b\f$ here. +4. The remaning values of Input(Transition) are for transition weights, +denoted as \f$w\f$ here. +5. Denote Input(Label) as \f$s\f$ here. + +The probability of a sequence \f$s\f$ of length \f$L\f$ is defined as: +\f$P(s) = (1/Z) \exp(a_{s_1} + b_{s_L} + + \sum_{l=1}^L x_{s_l} + + \sum_{l=2}^L w_{s_{l-1},s_l})\f$ +where \f$Z\f$ is a normalization value so that the sum of \f$P(s)\f$ over +all possible sequences is \f$1\f$, and \f$x\f$ is the emission feature weight +to the linear chain CRF. + +Finally, the linear chain CRF operator outputs the logarithm of the conditional +likelihood of each training sample in a mini-batch. + +NOTE: +1. The feature function for a CRF is made up of the emission features and the +transition features. The emission feature weights are NOT computed in +this operator. They MUST be computed first before this operator is called. + +2. Because this operator performs global normalization over all possible +sequences internally, it expects UNSCALED emission feature weights. +Please do not call this op with the emission feature being output of any +nonlinear activation. + +3. The 2nd dimension of Input(Emission) MUST be equal to the tag number. + +)DOC"); + } +}; + +class LinearChainCRFOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Emission"), + "Input(Emission) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Transition"), + "Input(Transition) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) should be not null."); + + PADDLE_ENFORCE(ctx->HasOutput("Alpha"), + "Output(Alpha) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("EmissionExps"), + "Output(EmissionExps) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("TransitionExps"), + "Output(TransitionExps) should be not null."); + PADDLE_ENFORCE(ctx->HasOutput("LogLikelihood"), + "Output(LogLikelihood) should be not null."); + + auto emission_dims = ctx->GetInputDim("Emission"); + PADDLE_ENFORCE_EQ(emission_dims.size(), 2UL, + "The Input(Emission) should be a 2-D tensor."); + PADDLE_ENFORCE(emission_dims[0], "An empty mini-batch is not allowed."); + + auto transition_dims = ctx->GetInputDim("Transition"); + PADDLE_ENFORCE_EQ(transition_dims.size(), 2UL, + "The Input(Transition) should be a 2-D tensor."); + PADDLE_ENFORCE_EQ( + transition_dims[0] - 2, transition_dims[1], + "An invalid dimension for the Input(Transition), which should " + "be a 2-D tensor with shape [(D + 2) x D]."); + PADDLE_ENFORCE_EQ( + emission_dims[1], transition_dims[1], + "The 2nd dimension of the Input(Emission) and the Input(Transition) " + "should be equal to the tag number."); + + auto label_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE(label_dims.size() == 2UL && label_dims[1] == 1UL, + "The Input(Label) should be a 2-D tensor with the 2nd " + "dimensions fixed to 1."); + PADDLE_ENFORCE_EQ( + emission_dims[0], label_dims[0], + "The height of Input(Emission) and the height of Input(Label) " + "should be the same."); + + ctx->SetOutputDim("Alpha", emission_dims); + ctx->SetOutputDim("EmissionExps", emission_dims); + ctx->SetOutputDim("TransitionExps", transition_dims); + // TODO(caoying) This is tricky. The 1st dimension of Output(LogLikelihood) + // is the sequence number in a mini-batch. The dimension set here should be + // resized to its correct size in the function Compute. Fix this once we can + // get LoD information in the InferShape interface. + ctx->SetOutputDim("LogLikelihood", {emission_dims[0], 1}); + } + + protected: + // Explicitly set that the data type of computation kernel of linear_chain_crf + // is determined by its input "Emission". + framework::DataType IndicateDataType( + const framework::ExecutionContext& ctx) const override { + return framework::ToDataType(ctx.Input("Emission")->type()); + } +}; + +class LinearChainCRFGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("EmissionExps"), + "Input(EmissionExps) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("TransitionExps"), + "Input(TransitionExps) should be not null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("LogLikelihood")), + "Input(LogLikelihood@GRAD) shoudl be not null."); + + auto emission_exps_dims = ctx->GetInputDim("EmissionExps"); + PADDLE_ENFORCE_EQ(emission_exps_dims.size(), 2UL, + "The Input(EmissionExps) should be a 2-D tensor."); + PADDLE_ENFORCE(emission_exps_dims[0], + "An empty mini-batch is not allowed."); + + auto transition_exps_dims = ctx->GetInputDim("TransitionExps"); + PADDLE_ENFORCE_EQ(transition_exps_dims.size(), 2UL, + "The Input(TransitionExps) should be a 2-D tensor."); + PADDLE_ENFORCE_EQ( + transition_exps_dims[0] - 2, transition_exps_dims[1], + "An invalid dimension for the Input(TransitionExps), which should " + "be a 2-D tensor with shape [(D + 2) x D]."); + PADDLE_ENFORCE_EQ( + emission_exps_dims[1], transition_exps_dims[1], + "The 2nd dimension of the Input(EmissionExps) and the " + "Input(TransitionExps) should be equal to the tag number."); + + auto label_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE(label_dims.size() == 2UL && label_dims[1] == 1UL, + "The Input(Label) should be a 2-D tensor with the 2nd " + "dimensions fixed to 1."); + PADDLE_ENFORCE_EQ( + emission_exps_dims[0], label_dims[0], + "The height of Input(EmissionExps) and the height of Input(Label) " + "should be the same."); + + if (ctx->HasOutput(framework::GradVarName("Emission"))) { + ctx->SetOutputDim(framework::GradVarName("Emission"), emission_exps_dims); + } + if (ctx->HasOutput(framework::GradVarName("Transition"))) { + ctx->SetOutputDim(framework::GradVarName("Transition"), + transition_exps_dims); + } + } + + protected: + // Explicitly set that the data type of output of the linear_chain_crf_grad + // operator is determined by its input: gradients of LogLikelihood. + framework::DataType IndicateDataType( + const framework::ExecutionContext& ctx) const override { + return framework::ToDataType( + ctx.Input(framework::GradVarName("LogLikelihood"))->type()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(linear_chain_crf, ops::LinearChainCRFOp, ops::LinearChainCRFOpMaker, + linear_chain_crf_grad, ops::LinearChainCRFGradOp); +REGISTER_OP_CPU_KERNEL( + linear_chain_crf, + ops::LinearChainCRFOpKernel, + ops::LinearChainCRFOpKernel); +REGISTER_OP_CPU_KERNEL( + linear_chain_crf_grad, + ops::LinearChainCRFGradOpKernel, + ops::LinearChainCRFGradOpKernel); diff --git a/paddle/operators/linear_chain_crf_op.cu b/paddle/operators/linear_chain_crf_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..6fc8995f4c2ce05f89ffb58129695113f89159fa --- /dev/null +++ b/paddle/operators/linear_chain_crf_op.cu @@ -0,0 +1,26 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/linear_chain_crf_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + linear_chain_crf, + ops::LinearChainCRFOpKernel, + ops::LinearChainCRFOpKernel); +REGISTER_OP_GPU_KERNEL( + linear_chain_crf_grad, + ops::LinearChainCRFGradOpKernel, + ops::LinearChainCRFGradOpKernel); diff --git a/paddle/operators/linear_chain_crf_op.h b/paddle/operators/linear_chain_crf_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ddf73981751798c72cef08f2dd5c87580b45aec3 --- /dev/null +++ b/paddle/operators/linear_chain_crf_op.h @@ -0,0 +1,543 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +template +static inline T NormalizeL1(T* x, size_t len) { + T sum = 0.; + for (size_t i = 0; i < len; ++i) sum += x[i]; + // (This comment is from the old LinearChainCRFLayer.) + // Right now, we just bet that sum won't be zero. If this really happens, we + // will figure out what should be done then. + PADDLE_ENFORCE(sum, + "The unnormalized probabilities of all possible unfinished " + "sequences must be greater than 0."); + T s = 1. / sum; + for (size_t i = 0; i < len; ++i) x[i] *= s; + return sum; +} + +template +struct ScalarMul { + explicit ScalarMul(const T& scalar) : scalar(scalar) {} + T operator()(const T& val) const { return val * scalar; } + + T scalar; +}; + +using framework::LoDTensor; +using framework::LoD; +using framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +template +class LinearChainCRFOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + // TODO(caoying) The checks related to LoD information should be + // moved into InferShape once after the InferShape is refactored. + PADDLE_ENFORCE_EQ(ctx.Input("Emission")->NumLevels(), 1UL, + "The Input(Emission) should be a sequence."); + PADDLE_ENFORCE_EQ(ctx.Input("Label")->NumLevels(), 1UL, + "The Input(Label) should be a sequence."); + auto in_lod = ctx.Input("Label")->lod(); + PADDLE_ENFORCE(in_lod.size(), "Input(Label) must be a sequence."); + const size_t level = 0; + const size_t seq_num = in_lod[level].size() - 1; + + // These local variables hold the inputs and outputs, garanteeing them on + // CPU memory, to provide a consistent reference. + // TODO(caoying) Fix this by moving all these local variables into the + // class's data members once we can profile the whole training process. + LoDTensor* emission_weights = nullptr; + LoDTensor emission_weight_tensor; + Tensor* transition_weights = nullptr; + Tensor transition_weight_tensor; + LoDTensor* label = nullptr; + LoDTensor label_tensor; + + Tensor* emission_exps = nullptr; + Tensor emission_exps_tensor; + Tensor* transition_exps = nullptr; + Tensor transition_exps_tensor; + Tensor* alpha = nullptr; + Tensor alpha_tensor; + Tensor* ll = nullptr; + Tensor ll_tensor; + + if (platform::is_gpu_place(ctx.GetPlace())) { + emission_weights = &emission_weight_tensor; + transition_weights = &transition_weight_tensor; + label = &label_tensor; + + CopyInputsToCpuMemory( + ctx.device_context(), *ctx.Input("Emission"), + *ctx.Input("Transition"), *ctx.Input("Label"), + emission_weights, transition_weights, label); + + emission_exps = &emission_exps_tensor; + emission_exps->Resize(emission_weights->dims()); + + transition_exps = &transition_exps_tensor; + transition_exps->Resize(transition_weights->dims()); + + alpha = &alpha_tensor; + alpha->Resize(ctx.Output("Alpha")->dims()); + + ll = &ll_tensor; + } else { + emission_weights = + const_cast(ctx.Input("Emission")); + transition_weights = const_cast(ctx.Input("Transition")); + label = const_cast(ctx.Input("Label")); + + emission_exps = ctx.Output("EmissionExps"); + transition_exps = ctx.Output("TransitionExps"); + alpha = ctx.Output("Alpha"); + ll = ctx.Output("LogLikelihood"); + } + + // Because the computation codes only runs on CPU, here the memory for all + // the outputs is FIXED to be allocated on the CPU memory. + emission_exps->mutable_data(platform::CPUPlace()); + transition_exps->mutable_data(platform::CPUPlace()); + alpha->mutable_data(platform::CPUPlace()); + + // Resize the output tensor to its correct dimension. + ll->Resize({static_cast(seq_num), 1}); + ll->mutable_data(platform::CPUPlace()); + + // Now, all the inputs and outputs should be on the CPU memory. + auto emission_dims = emission_weights->dims(); + const size_t batch_size = emission_dims[0]; + const size_t tag_num = emission_dims[1]; + + Tensor emission_row_max; + emission_row_max.mutable_data( + framework::make_ddim({static_cast(batch_size), 1}), + platform::CPUPlace()); + + auto place = ctx.GetEigenDevice(); + auto x = EigenMatrix::From(*emission_weights); + auto x_row_max = EigenMatrix::From(emission_row_max); + x_row_max.device(place) = + x.maximum(Eigen::DSizes(1)) + .reshape(Eigen::DSizes(int(batch_size), 1)); + + auto x_exps = EigenMatrix::From(*emission_exps); + x_exps.device(place) = + (x - x_row_max.broadcast(Eigen::DSizes(1, tag_num))).exp(); + + auto w = EigenMatrix::From(*transition_weights); + auto w_exps = EigenMatrix::From(*transition_exps); + w_exps.device(place) = w.exp(); + + T* log_likelihood = ll->data(); + for (size_t i = 0; i < seq_num; ++i) { + int start_pos = static_cast(in_lod[level][i]); + int end_pos = static_cast(in_lod[level][i + 1]); + if (end_pos == start_pos) { + // If an empty input sequence is given, pad 0 for its cost. + log_likelihood[i] = 0.; + continue; + } + + const Tensor one_seq = emission_weights->Slice(start_pos, end_pos); + Tensor one_seq_row_max = emission_row_max.Slice(start_pos, end_pos); + Tensor one_seq_exps = emission_exps->Slice(start_pos, end_pos); + const Tensor one_seq_label = label->Slice(start_pos, end_pos); + Tensor one_seq_alpha = alpha->Slice(start_pos, end_pos); + + log_likelihood[i] = ForwardOneSequence( + one_seq, one_seq_row_max, one_seq_exps, *transition_weights, + *transition_exps, one_seq_label, &one_seq_alpha); + } + + if (platform::is_gpu_place(ctx.GetPlace())) { + CopyOutputsToGpuMemory( + ctx.device_context(), *emission_exps, *transition_exps, *alpha, *ll, + ctx.Output("EmissionExps"), + ctx.Output("TransitionExps"), ctx.Output("Alpha"), + ctx.Output("LogLikelihood")); + } + }; + + private: + void CopyInputsToCpuMemory(const platform::DeviceContext& ctx, + const LoDTensor& emission_weights_src, + const Tensor& transition_weights_src, + const LoDTensor& label_src, + LoDTensor* emission_weights_dst, + Tensor* transition_weights_dst, + LoDTensor* label_dst) const { + // Copy the inputs from GPU memory to CPU memory if this operators runs on + // GPU device. + auto copyLoDTensor = [](const platform::DeviceContext& ctx, + const LoDTensor& src, LoDTensor* dst) { + dst->mutable_data(src.dims(), platform::CPUPlace()); + dst->CopyFrom(src, platform::CPUPlace(), ctx); + }; + + copyLoDTensor(ctx, emission_weights_src, emission_weights_dst); + copyLoDTensor(ctx, label_src, label_dst); + + transition_weights_dst->mutable_data(transition_weights_src.dims(), + platform::CPUPlace()); + transition_weights_dst->CopyFrom(transition_weights_src, + platform::CPUPlace(), ctx); + } + + void CopyOutputsToGpuMemory(const platform::DeviceContext& ctx, + const Tensor& emission_exps_src, + const Tensor& transition_exps_src, + const Tensor& alpha_src, const Tensor& ll_src, + Tensor* emission_exps_dst, + Tensor* transition_exps_dst, Tensor* alpha_dst, + Tensor* ll_dst) const { + // Copy the forward results from CPU memory to GPU memory if this + // operators runs on GPU device. + auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor& src, + Tensor* dst) { + dst->mutable_data(platform::GPUPlace()); + dst->CopyFrom(src, platform::GPUPlace(), ctx); + }; + copyTensor(ctx, emission_exps_src, emission_exps_dst); + copyTensor(ctx, transition_exps_src, transition_exps_dst); + copyTensor(ctx, alpha_src, alpha_dst); + copyTensor(ctx, ll_src, ll_dst); + } + + T ForwardOneSequence(const Tensor& emission, const Tensor& emission_row_max, + const Tensor& emission_exps, const Tensor& trans_weights, + const Tensor& trans_weight_exps, const Tensor& label, + Tensor* alpha) const { + const T* x = emission.data(); + const T* x_row_max = emission_row_max.data(); + const T* x_exps = emission_exps.data(); + const T* w = trans_weights.data(); + const T* w_exps = trans_weight_exps.data(); + T* alpha_value = alpha->data(); + + auto x_dims = emission.dims(); + const size_t seq_length = x_dims[0]; + const size_t tag_num = x_dims[1]; + // The 1st row of w are transition weights for start mask. + // The 2nd row of w are transition weights for end mask. + // Transition weights between other tags begin from the 3rd row of w. + const size_t state_trans_base_idx = 2; + + for (size_t i = 0; i < tag_num; ++i) { + alpha_value[i] = w_exps[i] * x_exps[i]; + } + T ll = -x_row_max[0] - std::log(NormalizeL1(alpha_value, tag_num)); + + for (size_t k = 1; k < seq_length; ++k) { + for (size_t i = 0; i < tag_num; ++i) { + T sum = 0.; + for (size_t j = 0; j < tag_num; ++j) { + sum += alpha_value[(k - 1) * tag_num + j] * // (*) + w_exps[(j + state_trans_base_idx) * tag_num + i]; + } + alpha_value[k * tag_num + i] = x_exps[k * tag_num + i] * sum; + } + // NormalizeL1 is to avoid underflow or overflow at (*). + ll -= x_row_max[k] + + std::log(NormalizeL1(alpha_value + k * tag_num, tag_num)); + } + T sum = 0.; + for (size_t i = 0; i < tag_num; ++i) { + sum += alpha_value[(seq_length - 1) * tag_num + i] * w_exps[tag_num + i]; + } + ll -= std::log(sum); + // Now ll is equal to -log(Z). + + const int* lbl = label.data(); + PADDLE_ENFORCE_LT( + static_cast(*std::max_element(lbl, lbl + seq_length)), tag_num, + "An invalid tag label that execesses the largest tag number."); + + // Calculate the nominator part, which depends on the label sequence. + ll += w[lbl[0]] /*start transition*/ + x[lbl[0]] + + w[tag_num + lbl[seq_length - 1]] /*end transition*/; + for (size_t k = 1; k < seq_length; ++k) { + ll += x[k * tag_num + lbl[k]] + + w[(lbl[k - 1] + state_trans_base_idx) * tag_num + lbl[k]]; + } + return -ll; + } +}; + +template +class LinearChainCRFGradOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + const size_t level = 0; // currently, only support sequence. + auto lod = ctx.Input("Label")->lod(); + PADDLE_ENFORCE(lod.size(), "Input(Label) must be a sequence."); + + // These local variables hold the inputs and outputs, garanteeing them on + // CPU memory, to provide a consistent reference. + // TODO(caoying) Fix this by moving all these local variables into the + // class's data members once we can profile the training process, or + // implementing a real GPU kernel for CRF. + Tensor* label = nullptr; + Tensor label_tensor; + Tensor* emission_exps = nullptr; + Tensor emission_exps_tensor; + Tensor* transition_exps = nullptr; + Tensor transition_exps_tensor; + Tensor* alpha = nullptr; + Tensor alpha_tensor; + Tensor ll_grad_tensor; + T* ll_grad = nullptr; + + Tensor* emission_grad = nullptr; + Tensor emission_grad_tensor; + Tensor* transition_grad = nullptr; + Tensor transition_grad_tensor; + + if (platform::is_gpu_place(ctx.GetPlace())) { + label = &label_tensor; + emission_exps = &emission_exps_tensor; + transition_exps = &transition_exps_tensor; + alpha = &alpha_tensor; + CopyInputsToCpuMemory( + ctx.device_context(), *ctx.Input("Label"), + *ctx.Input("EmissionExps"), + *ctx.Input("TransitionExps"), *ctx.Input("Alpha"), + *ctx.Input(framework::GradVarName("LogLikelihood")), label, + emission_exps, transition_exps, alpha, &ll_grad_tensor); + ll_grad = ll_grad_tensor.data(); + + if (ctx.Output(framework::GradVarName("Emission"))) { + emission_grad = &emission_grad_tensor; + emission_grad->Resize(emission_exps->dims()); + } + + if (ctx.Output(framework::GradVarName("Transition"))) { + transition_grad = &transition_grad_tensor; + transition_grad->Resize(transition_exps->dims()); + } + } else { + label = const_cast(ctx.Input("Label")); + emission_exps = const_cast(ctx.Input("EmissionExps")); + transition_exps = + const_cast(ctx.Input("TransitionExps")); + alpha = const_cast(ctx.Input("Alpha")); + ll_grad = const_cast( + ctx.Input(framework::GradVarName("LogLikelihood"))) + ->data(); + + emission_grad = ctx.Output(framework::GradVarName("Emission")); + transition_grad = + ctx.Output(framework::GradVarName("Transition")); + } + + // TODO(caoying) Fix this constraint. When the Input(Emission) is from the + // data reader operator, it can have no gradients. + PADDLE_ENFORCE(emission_grad, "Output(Emission@Grad) should not be null."); + emission_grad->mutable_data(platform::CPUPlace()); + if (transition_grad) { + transition_grad->mutable_data(platform::CPUPlace()); + math::SetConstant()(ctx.device_context(), + transition_grad, 0.); + } + // Now, all the inputs and outputs should be on the CPU memory. + + auto emission_dims = emission_exps->dims(); + // Beta is the memo table used in dynamic programming to calculate the + // backwark vectors. For a backward vector i (the i-th row of beta), it + // captures the unnormalized probabilities of partial sequences starting + // at position i. + Tensor beta; + beta.mutable_data(emission_dims, platform::CPUPlace()); + + for (size_t i = 0; i < lod[level].size() - 1; ++i) { + int start_pos = static_cast(lod[level][i]); + int end_pos = static_cast(lod[level][i + 1]); + if (end_pos == start_pos) continue; + + const Tensor one_seq_emission_exps = + emission_exps->Slice(start_pos, end_pos); + const Tensor one_seq_label = label->Slice(start_pos, end_pos); + const Tensor one_seq_alpha = alpha->Slice(start_pos, end_pos); + Tensor one_seq_beta = beta.Slice(start_pos, end_pos); + Tensor one_seq_emission_grad = emission_grad->Slice(start_pos, end_pos); + + BackwardOneSequence(ctx.device_context(), ll_grad[i], + one_seq_emission_exps, *transition_exps, + one_seq_alpha, one_seq_label, &one_seq_beta, + transition_grad, &one_seq_emission_grad); + } + + if (platform::is_gpu_place(ctx.GetPlace())) { + CopyOutputsToGpuMemory( + ctx.device_context(), emission_grad, transition_grad, + ctx.Output(framework::GradVarName("Emission")), + ctx.Output(framework::GradVarName("Transition"))); + } + }; + + private: + void CopyInputsToCpuMemory(const platform::DeviceContext& ctx, + const LoDTensor& label_src, + const Tensor& emission_exps_src, + const Tensor& transition_exps_src, + const Tensor& alpha_src, const Tensor& ll_grad_src, + Tensor* label_dst, Tensor* emission_exps_dst, + Tensor* transition_exps_dst, Tensor* alpha_dst, + Tensor* ll_grad_dst) const { + // Copy the inputs from GPU memory to CPU memory when this operators runs on + // GPU device. + label_dst->mutable_data(label_src.dims(), platform::CPUPlace()); + label_dst->CopyFrom(label_src, platform::CPUPlace(), ctx); + + auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor& src, + Tensor* dst) { + dst->mutable_data(src.dims(), platform::CPUPlace()); + dst->CopyFrom(src, platform::CPUPlace(), ctx); + }; + copyTensor(ctx, emission_exps_src, emission_exps_dst); + copyTensor(ctx, transition_exps_src, transition_exps_dst); + copyTensor(ctx, alpha_src, alpha_dst); + copyTensor(ctx, ll_grad_src, ll_grad_dst); + } + + void CopyOutputsToGpuMemory(const platform::DeviceContext& ctx, + const Tensor* emission_grad_src, + const Tensor* transition_grad_src, + Tensor* emission_grad_dst, + Tensor* transition_grad_dst) const { + // Copy the backward results from CPU memory to GPU + // memory if this operators runs on GPU device. + auto copyTensor = [](const platform::DeviceContext& ctx, const Tensor* src, + Tensor* dst) { + if (src && dst) { + dst->mutable_data(platform::GPUPlace()); + dst->CopyFrom(*src, platform::GPUPlace(), ctx); + } + }; + copyTensor(ctx, emission_grad_src, emission_grad_dst); + copyTensor(ctx, transition_grad_src, transition_grad_dst); + } + + void BackwardOneSequence(const platform::DeviceContext& ctx, const T ll_grad, + const Tensor& emission_exps, + const Tensor& transition_exps, const Tensor& alpha, + const Tensor& label, Tensor* beta, + Tensor* transition_grad, + Tensor* emission_grad) const { + const T* w_exps = transition_exps.data(); + const T* x_exps = emission_exps.data(); + const int* label_value = label.data(); + T* beta_value = beta->data(); + + auto x_dims = emission_exps.dims(); + const size_t seq_length = x_dims[0]; + const size_t tag_num = x_dims[1]; + const size_t state_trans_base_idx = 2; + + // Calculate the backward vectors: beta. + // First, calculate the initialition state. + for (size_t i = 0; i < tag_num; ++i) { + beta_value[(seq_length - 1) * tag_num + i] = w_exps[tag_num + i]; + } + NormalizeL1(beta_value + (seq_length - 1) * tag_num, tag_num); + for (int k = static_cast(seq_length) - 2; k >= 0; --k) { + for (size_t i = 0; i < tag_num; ++i) { + T sum = 0.; + for (size_t j = 0; j < tag_num; ++j) { + sum += w_exps[(i + state_trans_base_idx) * tag_num + j] * // (**) + x_exps[(k + 1) * tag_num + j] * + beta_value[(k + 1) * tag_num + j]; + } + beta_value[k * tag_num + i] = sum; + } + // NormalizeL1 is to avoid underflow or overflow at (**). + NormalizeL1(beta_value + k * tag_num, tag_num); + } + + auto x_grad_mat = EigenMatrix::From(*emission_grad); + auto alpha_mat = EigenMatrix::From(alpha); + auto beta_mat = EigenMatrix::From(*beta); + + auto* place = ctx.GetEigenDevice(); + auto prob = alpha_mat * beta_mat; + auto row_sum = prob.sum(Eigen::DSizes(1)) + .reshape(Eigen::DSizes(seq_length, 1)) + .broadcast(Eigen::DSizes(1, tag_num)); + x_grad_mat.device(*place) = + (prob / row_sum).unaryExpr(ScalarMul(ll_grad)); + + for (size_t k = 0; k < seq_length; ++k) { + x_grad_mat(k, label_value[k]) -= static_cast(ll_grad); + } + + if (transition_grad) { + T* trans_grad = transition_grad->data(); + for (size_t k = 0; k < tag_num; ++k) { + // Do not multiply by the output gradient here, because x_grad_mat has + // alrealy done this. + trans_grad[k] += x_grad_mat(/*from start state*/ 0, k); + trans_grad[tag_num + k] += + x_grad_mat(/*to end state*/ seq_length - 1, k); + } + + auto x_exps_mat = EigenMatrix::From(emission_exps); + + // TODO(caoying): Fix this to avoid using this local variable if we can + // profile the training process. + Tensor tmp; + tmp.mutable_data(beta->dims(), platform::CPUPlace()); + auto tmp_mat = EigenMatrix::From(tmp); + auto prob = beta_mat * x_exps_mat; + auto row_sum = prob.sum(Eigen::DSizes(1)) + .reshape(Eigen::DSizes(seq_length, 1)) + .broadcast(Eigen::DSizes(1, tag_num)); + tmp_mat.device(*place) = prob / row_sum; + + for (size_t k = 1; k < seq_length; ++k) { + T sum = 0.; + for (size_t i = 0; i < tag_num; ++i) { + for (size_t j = 0; j < tag_num; ++j) { + sum += w_exps[(i + state_trans_base_idx) * tag_num + j] * // (**) + alpha_mat(k - 1, i) * tmp_mat(k, j); + } + } + sum = 1. / sum; + for (size_t i = 0; i < tag_num; ++i) { + for (size_t j = 0; j < tag_num; ++j) { + trans_grad[(i + state_trans_base_idx) * tag_num + j] += + sum * w_exps[(i + state_trans_base_idx) * tag_num + j] * + alpha_mat(k - 1, i) * tmp_mat(k, j) * ll_grad; + } + } + trans_grad[(label_value[k - 1] + state_trans_base_idx) * tag_num + + label_value[k]] -= static_cast(ll_grad); + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/load_op.cc b/paddle/operators/load_op.cc index 2d4eff0c35af520dd27b9eb197937026a8fbdff9..b71a33a6b1ce80b545e6d7a4020dafc941dc55d2 100644 --- a/paddle/operators/load_op.cc +++ b/paddle/operators/load_op.cc @@ -115,14 +115,18 @@ class LoadOpProtoMaker : public framework::OpProtoAndCheckerMaker { LoadOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "The tensor need to be loaded"); - AddComment(R"DOC(Load Operator -Load operator will load a tensor variable from disk file. -)DOC"); + AddOutput("Out", "(Tensor) The tensor need to be loaded"); AddAttr("file_path", + "(string) " "Variable will be loaded from \"file_path\".") .AddCustomChecker( [](const std::string &path) { return !path.empty(); }); + AddComment(R"DOC( +Load Operator. + +Load operator will load a tensor variable from disk file. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/lod_rank_table_op.cc b/paddle/operators/lod_rank_table_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..be198951c241dc5e5587c8a2b8d94f67173d2b2a --- /dev/null +++ b/paddle/operators/lod_rank_table_op.cc @@ -0,0 +1,80 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/op_registry.h" +namespace paddle { +namespace operators { + +class LoDRankTableOp : public framework::OperatorBase { + public: + LoDRankTableOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto x = scope.FindVar(Input("X"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + out->Reset(x.lod(), static_cast(Attr("level"))); + } +}; + +class LoDRankTableOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDRankTableOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor) input lod tensor, must contain lod information."); + AddOutput("Out", "(LoDRankTable) The rank table of specific level."); + AddAttr("level", "(int) the specific lod level to rank.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment(R"DOC(Create LoDRanTable by LoDTensor + +LoD Rank Table stores the `level` of `lod` which is ordered by sequence +length in descending order. It is useful when implement dynamic RNN and is +shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +output operators. +)DOC"); + } +}; + +class LoDRankTableInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), "LoDRankTable must has input X"); + } +}; + +class LoDRankTableInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind &op_desc, + framework::BlockDescBind *block) const override { + for (auto &o : op_desc.Output("Out")) { + block->Var(o)->SetType(framework::VarDesc::LOD_RANK_TABLE); + } + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(lod_rank_table, paddle::operators::LoDRankTableOp, + paddle::operators::LoDRankTableOpProtoMaker, + paddle::operators::LoDRankTableInferShape, + paddle::operators::LoDRankTableInferVarType, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc index 8fdd42352e5e6857e4bf0e4645f82c8e2fcdc6fd..2163c8ce4e5d75d5934c08f59a47bad9553f0c8b 100644 --- a/paddle/operators/lookup_table_op.cc +++ b/paddle/operators/lookup_table_op.cc @@ -43,7 +43,7 @@ class LookupTableOp : public framework::OperatorWithKernel { protected: framework::DataType IndicateDataType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("W")->type()); + return framework::ToDataType(ctx.Input("W")->type()); } }; @@ -53,21 +53,27 @@ class LookupTableOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("W", - "An input represents embedding tensors," - " which is a learnable parameter."); + "An input represents embedding tensors, " + "which is a learnable parameter."); AddInput("Ids", - "An input with type int32 or int64" - "contains the ids to be looked up in W." - "Ids must be a column vector with rank = 2." - "The 2nd dimension size must be 1"); - AddOutput("Out", "The lookup results, which have the same type with W."); - AddAttr("is_sparse", "Sparse update").SetDefault(false); + "An input with type int32 or int64 " + "contains the ids to be looked up in W. " + "Ids must be a column vector with rank = 2. " + "The 2nd dimension size must be 1."); + AddOutput("Out", "The lookup results, which have the same type as W."); + AddAttr("is_sparse", + "(boolean, default false) " + "Sparse update") + .SetDefault(false); AddComment(R"DOC( +Lookup Table Operator. + This operator is used to perform lookups on the parameter W, then concatenated into a dense tensor. -The input `Ids` can carry the LoD (Level of Details) information, -or not. And the output only shares the LoD with input `Ids`. +The input Ids can carry the LoD (Level of Details) information, +or not. And the output only shares the LoD information with input Ids. + )DOC"); } }; @@ -93,7 +99,7 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { protected: framework::DataType IndicateDataType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("W")->type()); + return framework::ToDataType(ctx.Input("W")->type()); } }; diff --git a/paddle/operators/lookup_table_op.cu b/paddle/operators/lookup_table_op.cu index 837b2a1f4c94f201c0ab498671f936aab6c7a811..c7ba1720662fe80c945f2b4aa19745e408d40948 100644 --- a/paddle/operators/lookup_table_op.cu +++ b/paddle/operators/lookup_table_op.cu @@ -61,16 +61,16 @@ template class LookupTableCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto table_t = context.Input("W"); - auto ids_t = context.Input("Ids"); - auto output_t = context.Output("Out"); + auto* table_t = context.Input("W"); + auto* ids_t = context.Input("Ids"); + auto* output_t = context.Output("Out"); size_t N = table_t->dims()[0]; size_t D = table_t->dims()[1]; size_t K = ids_t->numel(); - auto ids = ids_t->data(); - auto table = table_t->data(); - auto output = output_t->mutable_data(context.GetPlace()); + auto* ids = ids_t->data(); + auto* table = table_t->data(); + auto* output = output_t->mutable_data(context.GetPlace()); dim3 threads(128, 8); dim3 grids(8, 1); @@ -87,9 +87,9 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& context) const override { bool is_sparse = context.Attr("is_sparse"); if (is_sparse) { - auto* ids = context.Input("Ids"); - auto* table = context.Input("W"); - auto* d_output = context.Input(framework::GradVarName("Out")); + auto* ids = context.Input("Ids"); + auto* table = context.Input("W"); + auto* d_output = context.Input(framework::GradVarName("Out")); auto* d_table = context.Output(framework::GradVarName("W")); auto* ids_data = ids->data(); @@ -116,12 +116,12 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { auto* d_output_data = d_output->data(); PADDLE_ENFORCE_EQ(d_table_value->dims(), d_output->dims()); memory::Copy(gpu_place, d_table_data, gpu_place, d_output_data, - d_output->numel(), stream); + d_output->numel() * sizeof(T), stream); } else { - auto ids_t = context.Input("Ids"); - auto d_output_t = context.Input(framework::GradVarName("Out")); - auto d_table_t = context.Output(framework::GradVarName("W")); + auto ids_t = context.Input("Ids"); + auto d_output_t = context.Input(framework::GradVarName("Out")); + auto d_table_t = context.Output(framework::GradVarName("W")); int N = d_table_t->dims()[0]; int D = d_table_t->dims()[1]; diff --git a/paddle/operators/lookup_table_op.h b/paddle/operators/lookup_table_op.h index 54067cd01d3ef35a050a3c2565ea19cb6520bcec..99b912163b71594340d8917645dff107fd208aea 100644 --- a/paddle/operators/lookup_table_op.h +++ b/paddle/operators/lookup_table_op.h @@ -19,22 +19,22 @@ namespace paddle { namespace operators { -using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; using SelectedRows = framework::SelectedRows; template class LookupTableKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto table_t = context.Input("W"); // float tensor - auto ids_t = context.Input("Ids"); // int tensor - auto output_t = context.Output("Out"); // float tensor + auto* table_t = context.Input("W"); // float tensor + auto* ids_t = context.Input("Ids"); // int tensor + auto* output_t = context.Output("Out"); // float tensor int N = table_t->dims()[0]; int D = table_t->dims()[1]; - auto ids = ids_t->data(); - auto table = table_t->data(); - auto output = output_t->mutable_data(context.GetPlace()); + auto* ids = ids_t->data(); + auto* table = table_t->data(); + auto* output = output_t->mutable_data(context.GetPlace()); for (int64_t i = 0; i < ids_t->numel(); ++i) { PADDLE_ENFORCE_LT(ids[i], N); PADDLE_ENFORCE_GE(ids[i], 0); @@ -49,9 +49,9 @@ class LookupTableGradKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& context) const override { bool is_sparse = context.Attr("is_sparse"); if (is_sparse) { - auto* ids = context.Input("Ids"); - auto* table = context.Input("W"); - auto* d_output = context.Input(framework::GradVarName("Out")); + auto* ids = context.Input("Ids"); + auto* table = context.Input("W"); + auto* d_output = context.Input(framework::GradVarName("Out")); auto* d_table = context.Output(framework::GradVarName("W")); auto* ids_data = ids->data(); @@ -76,10 +76,10 @@ class LookupTableGradKernel : public framework::OpKernel { PADDLE_ENFORCE_EQ(d_table_value->dims(), d_output->dims()); memcpy(d_table_data, d_output_data, sizeof(T) * d_output->numel()); } else { - auto* ids = context.Input("Ids"); - auto* d_output = context.Input(framework::GradVarName("Out")); - auto* d_table = context.Output(framework::GradVarName("W")); - auto* table = context.Input("W"); + auto* ids = context.Input("Ids"); + auto* d_output = context.Input(framework::GradVarName("Out")); + auto* d_table = context.Output(framework::GradVarName("W")); + auto* table = context.Input("W"); auto* ids_data = ids->data(); auto ids_dim = ids->dims(); @@ -90,11 +90,13 @@ class LookupTableGradKernel : public framework::OpKernel { auto* d_output_data = d_output->data(); auto* d_table_data = d_table->mutable_data(context.GetPlace()); + memset(d_table_data, 0, d_table->numel() * sizeof(T)); + for (int64_t i = 0; i < ids->numel(); ++i) { PADDLE_ENFORCE_LT(ids_data[i], N); PADDLE_ENFORCE_GE(ids_data[i], 0); for (int j = 0; j < D; ++j) { - d_table_data[ids_data[i] * D + j] = d_output_data[i * D + j]; + d_table_data[ids_data[i] * D + j] += d_output_data[i * D + j]; } } } diff --git a/paddle/operators/lrn_op.cc b/paddle/operators/lrn_op.cc index 89ea6bfdbd9b78dd0a81fd5ba465d09549162eb5..00392b7967d020a7951a16a7850a2f08735baeb8 100644 --- a/paddle/operators/lrn_op.cc +++ b/paddle/operators/lrn_op.cc @@ -45,72 +45,70 @@ class LRNOpMaker : public framework::OpProtoAndCheckerMaker { public: LRNOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", R"DOC( - (Tensor) The input of LRN operator. It must be a 4D tenor with NCHW format. - )DOC"); - + AddInput("X", + "(Tensor) The input of LRN operator. " + "It must be a 4D tenor with NCHW format."); AddOutput("Out", "(Tensor) The output of LRN operator, which is also the 4D " "tensor with NCHW format."); - AddOutput("MidOut", R"Doc( -(Tensor)Middle result of lrn op.It's computed in forward process -and also used in backward process. - )Doc"); - - AddAttr("n", R"DOC( -(int, default 5)n is “adjacent” kernel maps at the same spatial position. - )DOC") + AddOutput("MidOut", + "(Tensor) Middle result of LRN operator. It's computed in " + "forward process and also used in backward process."); + + AddAttr("n", + "(int default 5) " + "n is the \"adjacent\" kernel that maps " + "at the same spatial position.") .SetDefault(5) .GreaterThan(0); - AddAttr("k", R"DOC( -(float, default 2.0)k is the bias. - )DOC") + AddAttr("k", + "(float, default 2.0) " + "k is the bias.") .SetDefault(2.0) .GreaterThan(0.0); - AddAttr("alpha", R"DOC( -(float, default 0.0001)alpha is the scale number. - )DOC") + AddAttr("alpha", + "(float, default 0.0001) " + "alpha is the scale number.") .SetDefault(0.0001) .GreaterThan(0.0); - AddAttr("beta", R"DOC( -(float, default 0.75)beta is the power number. - )DOC") + AddAttr("beta", + "(float, default 0.75) " + "beta is the power number.") .SetDefault(0.75) .GreaterThan(0.0); AddComment(R"DOC( - Local Response Normalization. - - This Function comes from the paper - "ImageNet Classification with Deep Convolutional Neural Networks". +Local Response Normalization Operator. - The original formula is: +This operator comes from the paper +"ImageNet Classification with Deep Convolutional Neural Networks". - Input(i, x, y) - Output(i, x, y) = ---------------------------------------------- - -- upper - (k + alpha * > (Input(j, x, y))^2) ^ (beta) - -- j = lower +The original formula is: - upper is `min(C, c + n/2)` - lower if `max(0, c - n/2)` +$$ +Output(i, x, y) = Input(i, x, y) / \left( +k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} +(Input(j, x, y))^2 +\right)^{\beta} +$$ - Function implementation: +Function implementation: - inputs and outpus is NCHW format, while input.shape.ndims() is equal 4. - And the meaning of each dimension(0-3) is respectively batch size, - feature maps, rows and columns. +Inputs and outpus are in NCHW format, while input.shape.ndims() equals 4. +And dimensions 0 ~ 3 represent batch size, feature maps, rows, +and columns, respectively. - Input and Output in the above formula is for each map(i) of one image, and - Input(i, x, y), Output(i, x, y) represents an element in an image. +Input and Output in the formula above is for each map(i) of one image, and +Input(i, x, y), Output(i, x, y) represents an element in an image. - C is the number of feature maps of one image, and n is a hyper-parameters - is configured when Function is initialized. The sum in the denominator - is the sum of the same position in the neighboring maps. - )DOC"); +C is the number of feature maps of one image. n is a hyper-parameter +configured when operator is initialized. The sum in the denominator +is the sum of the same positions in the neighboring maps. + +)DOC"); } }; diff --git a/paddle/operators/lstm_op.cc b/paddle/operators/lstm_op.cc index 0a089b7c2dc1e05224525bc4fe5399ec39036d01..fdf52cf424d1b2727982e6e76f0f824915d84968 100644 --- a/paddle/operators/lstm_op.cc +++ b/paddle/operators/lstm_op.cc @@ -21,7 +21,6 @@ class LSTMOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; - protected: void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Input"), "Input(Input) of LSTM should not be null."); @@ -29,9 +28,13 @@ class LSTMOp : public framework::OperatorWithKernel { "Output(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Cell"), "Output(Cell) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchGate"), + "Output(BatchGate) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchCellPreAct"), + "Output(BatchGate) of LSTM should not be null."); - auto x_dims = ctx->GetInputDim("Input"); - PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2."); + auto in_dims = ctx->GetInputDim("Input"); + PADDLE_ENFORCE_EQ(in_dims.size(), 2, "Input(X)'s rank must be 2."); if (ctx->HasInput("H0")) { PADDLE_ENFORCE(ctx->HasInput("C0"), @@ -44,7 +47,7 @@ class LSTMOp : public framework::OperatorWithKernel { "should be the same."); } - int frame_size = x_dims[1] / 4; + int frame_size = in_dims[1] / 4; auto w_dims = ctx->GetInputDim("Weight"); PADDLE_ENFORCE_EQ(w_dims.size(), 2, "The rank of Input(Weight) should be 2."); @@ -71,12 +74,21 @@ class LSTMOp : public framework::OperatorWithKernel { "4 * %d if disable peepholes connection", frame_size); } - ctx->SetOutputDim("Hidden", {x_dims[0], frame_size}); - ctx->SetOutputDim("Cell", {x_dims[0], frame_size}); - ctx->SetOutputDim("BatchGate", x_dims); + framework::DDim out_dims({in_dims[0], frame_size}); + ctx->SetOutputDim("Hidden", out_dims); + ctx->SetOutputDim("Cell", out_dims); + ctx->SetOutputDim("BatchGate", in_dims); + ctx->SetOutputDim("BatchCellPreAct", out_dims); ctx->ShareLoD("Input", "Hidden"); ctx->ShareLoD("Input", "Cell"); } + + protected: + framework::DataType IndicateDataType( + const framework::ExecutionContext& ctx) const override { + return framework::ToDataType( + ctx.Input("Input")->type()); + } }; class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { @@ -86,16 +98,18 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Input", "(LoDTensor) the first input is a LodTensor, which support " "variable-time length input sequence. The underlying tensor in " - "this LoDTensor is a matrix with shape (T X 4D), where, T is the " + "this LoDTensor is a matrix with shape (T X 4D), where T is the " "total time steps in this mini-batch, D is the hidden size."); AddInput("H0", "(Tensor, optional) the initial hidden state is an optional " "input. This is a tensor with shape (N x D), where N is the " - "batch size, D is the hidden size."); + "batch size and D is the hidden size.") + .AsDispensable(); AddInput("C0", "(Tensor, optional) the initial cell state is an optional " "input. This is a tensor with shape (N x D), where N is the " - "batch size. `H0` and `C0` can be NULL but only at the same time"); + "batch size. `H0` and `C0` can be NULL but only at the same time") + .AsDispensable(); AddInput("Weight", "(Tensor) the learnable hidden-hidden weights." " - The shape is (D x 4D), where D is the hidden size. " @@ -109,91 +123,93 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { " - Bias = {b_c, b_i, b_f, b_o}." "2. `usePeepholes = True` " " - The shape is (1 x 7D). " - " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}."); + " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}.") + .AsDispensable(); + AddOutput("Hidden", + "(LoDTensor) the hidden state of LSTM operator. " + "The shape is (T x D), and lod is the same with the `Input`."); + AddOutput("Cell", + "(LoDTensor) the cell state of LSTM operator. " + "The shape is (T x D), and lod is the same with the `Input`."); AddOutput("BatchGate", "(LoDTensor) This LoDTensor contains input gate, forget gate " "and output gate after the nonlinear computation. This " - "LoDTensor has the same shape with the reorganized input, which " - "was also be called batch input. The LoD size is 2. The first " + "LoDTensor has the same shape as the reorganized input, which " + "is also be called batch input. The LoD size is 2. The first " "LoD is the batch offsets and the second LoD contains the " "indexes, which denote the position of reorganized sequence " "in the raw input.") .AsIntermediate(); - AddOutput("Hidden", - "(LoDTensor) the hidden state lod tensor of LSTM operator. " - "The shape and lod is the same with the `Input`."); - AddOutput("Cell", - "(LoDTensor) the cell state lod tensor of LSTM operator. " - "The shape and lod is the same with the `Input`."); + AddOutput("BatchCellPreAct", + "(LoDTensor) This LoDTensor is obtained in the forward and used " + "in the backward.") + .AsIntermediate(); AddAttr("usePeepholes", - "(bool, defalut: True) " + "(bool, default True) " "whether to enable diagonal/peephole connections.") .SetDefault(true); AddAttr("isReverse", - "(bool, defalut: False) " + "(bool, default False) " "whether to compute reversed LSTM.") .SetDefault(false); AddAttr( "gateActivation", - "(string, default: sigmoid)" + "(string, default sigmoid)" "The activation for input gate, forget gate and output " "gate, `sigmoid` by default.") .SetDefault("sigmoid"); AddAttr("cellActivation", - "(string, default: tanh)" + "(string, default tanh)" "The activation for cell output, `tanh` by defalut.") .SetDefault("tanh"); AddAttr("candidateActivation", - "(string, default: tanh)" + "(string, default tanh)" "The activation for candidate hidden state, " "`tanh` by default.") .SetDefault("tanh"); - AddComment(R"DOC(Long-Short Term Memory (LSTM) Operator + AddComment(R"DOC( +Long-Short Term Memory (LSTM) Operator. -The defalut implementation is diagonal/peephole connection [1], the formula is -as follows +The defalut implementation is diagonal/peephole connection +(https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: - i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) +$$ +i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) \\ - f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) +f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) \\ - \tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) +\tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) \\ - o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) +o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) \\ - c_t = f_t ⊙ c_{t-1} + i_t ⊙ \tilde{c_t} +c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ - h_t = o_t ⊙ act_h(c_t) +h_t = o_t \odot act_h(c_t) +$$ where the W terms denote weight matrices (e.g. \f$W_{xi}\f$ is the matrix of weights from the input gate to the input), \f$W_{ic}, W_{fc}, W_{oc}\f$ -are diagonal weight matrices for peephole connections. In our implenmention, -We use vectors to reprenset these diagonal weight matrices. The b terms +are diagonal weight matrices for peephole connections. In our implementation, +we use vectors to reprenset these diagonal weight matrices. The b terms denote bias vectors (\f$b_i\f$ is the input gate bias vector), \f$\sigma\f$ -is the non-line actications, such as logistic sigmoid function, and -\f$i, f, o\f$ and \f$c\f$ are respectively the input gate, forget gate, -output gate and cell activation vectors, all of which are the same size as +is the non-line activations, such as logistic sigmoid function, and +\f$i, f, o\f$ and \f$c\f$ are the input gate, forget gate, output gate, +and cell activation vectors, respectively, all of which have the same size as the cell output activation vector \f$h\f$. -The ⊙ is the element-wise product of the vectors, \f$act_g\f$ and \f$act_h\f$ -are the cell input and cell output activation functions, `tanh` is usually +The \f$\odot\f$ is the element-wise product of the vectors. \f$act_g\f$ and \f$act_h\f$ +are the cell input and cell output activation functions and `tanh` is usually used for them. \f$\tilde{c_t}\f$ is also called candidate hidden state, which is computed based on the current input and the previous hidden state. -Set `usePeepholes` False to disable peephole connection [2]. The formula +Set usePeepholes False to disable peephole connection +(http://www.bioinf.jku.at/publications/older/2604.pdf). The formula is omitted here. -@note These \f$W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}\f$ -operations on the input x_{t} were NOT included in this operator. +Note that these \f$W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}\f$ +operations on the input \f$x_{t}\f$ are NOT included in this operator. Users can choose to use fully-connect operator before LSTM operator. -[1] Hasim Sak, Andrew Senior, and Francoise Beaufays. Long short-term memory -recurrent neural network architectures for large scale acoustic modeling. -INTERSPEECH, 2014. - -[2] S. Hochreiter and J. Schmidhuber. Long Short-Term Memory. -Neural Computation, 9(8):1735-1780, 1997. - )DOC"); } }; @@ -202,15 +218,37 @@ class LSTMGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; - protected: void InferShape(framework::InferShapeContext* ctx) const override { - PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Hidden")), - "Input(Hidden@GRAD) should not be null"); - PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Cell")), - "Input(Cell@GRAD) should not be null"); - ctx->SetOutputDim(framework::GradVarName("Weight"), - ctx->GetInputDim("Weight")); - ctx->SetOutputDim(framework::GradVarName("Bias"), ctx->GetInputDim("Bias")); + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(Input) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Hidden"), + "Input(Hidden) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Cell"), + "Input(Cell) of LSTM should not be null."); + + PADDLE_ENFORCE(ctx->HasInput("BatchGate"), + "Input(BatchGate) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("BatchCellPreAct"), + "Input(BatchGate) of LSTM should not be null."); + + auto in_g_name = framework::GradVarName("Input"); + if (ctx->HasOutput(in_g_name)) + ctx->SetOutputDim(in_g_name, ctx->GetInputDim("Input")); + + auto w_g_name = framework::GradVarName("Weight"); + if (ctx->HasOutput(w_g_name)) + ctx->SetOutputDim(w_g_name, ctx->GetInputDim("Weight")); + + auto b_g_name = framework::GradVarName("Bias"); + if (ctx->HasOutput(b_g_name)) + ctx->SetOutputDim(b_g_name, ctx->GetInputDim("Bias")); + } + + protected: + framework::DataType IndicateDataType( + const framework::ExecutionContext& ctx) const override { + return framework::ToDataType( + ctx.Input("Input")->type()); } }; diff --git a/paddle/operators/lstm_op.h b/paddle/operators/lstm_op.h index 0af5694c48fcb4437e3acd422606de013bb2e145..af088b80b4283cf221a1dff74546d73d977fada3 100644 --- a/paddle/operators/lstm_op.h +++ b/paddle/operators/lstm_op.h @@ -21,8 +21,9 @@ limitations under the License. */ namespace paddle { namespace operators { -using framework::LoDTensor; -using framework::Tensor; +using LoDTensor = framework::LoDTensor; +using Tensor = framework::Tensor; + template using EigenMatrix = framework::EigenMatrix; @@ -31,15 +32,15 @@ template class LSTMKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto* input = ctx.Input("Input"); - auto* weight = ctx.Input("Weight"); - auto* bias = ctx.Input("Bias"); + auto* input = ctx.Input("Input"); + auto* weight = ctx.Input("Weight"); + auto* bias = ctx.Input("Bias"); - auto* batch_gate = ctx.Output("BatchGate"); + auto* batch_gate = ctx.Output("BatchGate"); batch_gate->mutable_data(ctx.GetPlace()); - auto* hidden_out = ctx.Output("Hidden"); + auto* hidden_out = ctx.Output("Hidden"); hidden_out->mutable_data(ctx.GetPlace()); - auto* cell_out = ctx.Output("Cell"); + auto* cell_out = ctx.Output("Cell"); cell_out->mutable_data(ctx.GetPlace()); // Now the function ShareLoD in InferShape is not implemented. @@ -49,7 +50,8 @@ class LSTMKernel : public framework::OpKernel { bool is_reverse = ctx.Attr("isReverse"); math::LoDTensor2BatchFunctor to_batch; - to_batch(ctx.device_context(), *input, *batch_gate, is_reverse); + auto& device_ctx = ctx.device_context(); + to_batch(device_ctx, *input, *batch_gate, true, is_reverse); auto in_dims = input->dims(); int frame_size = static_cast(in_dims[1] / 4); @@ -69,17 +71,26 @@ class LSTMKernel : public framework::OpKernel { } math::LstmMetaValue lstm_value; - T* bias_data = const_cast(bias->data()); - // the code style in LstmMetaValue will be updated later. - lstm_value.checkIg = bias_data + 4 * frame_size; - lstm_value.checkFg = lstm_value.checkIg + frame_size; - lstm_value.checkOg = lstm_value.checkFg + frame_size; + if (bias) { + T* bias_data = const_cast(bias->data()); + // the code style in LstmMetaValue will be updated later. + + lstm_value.checkIg = bias_data + 4 * frame_size; + lstm_value.checkFg = lstm_value.checkIg + frame_size; + lstm_value.checkOg = lstm_value.checkFg + frame_size; + } else { + lstm_value.checkIg = nullptr; + lstm_value.checkFg = nullptr; + lstm_value.checkOg = nullptr; + } lstm_value.prevStateValue = nullptr; - framework::LoDTensor batch_out, batch_cell, batch_cell_pre_act; - batch_out.mutable_data(dims, ctx.GetPlace()); + // Use the local variable as here. + LoDTensor batch_hidden, batch_cell; + auto* batch_cell_pre_act = ctx.Output("BatchCellPreAct"); + batch_hidden.mutable_data(dims, ctx.GetPlace()); batch_cell.mutable_data(dims, ctx.GetPlace()); - batch_cell_pre_act.mutable_data(dims, ctx.GetPlace()); + batch_cell_pre_act->mutable_data(dims, ctx.GetPlace()); auto batch_starts = batch_gate->lod()[0]; size_t num_batch = batch_starts.size() - 1; @@ -92,18 +103,18 @@ class LSTMKernel : public framework::OpKernel { int bend = static_cast(batch_starts[n + 1]); Tensor gate_t = batch_gate->Slice(bstart, bend); - Tensor out_t = batch_out.Slice(bstart, bend); + Tensor out_t = batch_hidden.Slice(bstart, bend); Tensor cell_t = batch_cell.Slice(bstart, bend); - Tensor cell_pre_act_t = batch_cell_pre_act.Slice(bstart, bend); + Tensor cell_pre_act_t = batch_cell_pre_act->Slice(bstart, bend); int cur_batch_size = bend - bstart; if (n != 0) { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; - auto pre_hidden_t = batch_out.Slice(pre_h_start, pre_h_end); - math::matmul(ctx.device_context(), pre_hidden_t, false, - *weight, false, static_cast(1.0), &gate_t, + auto pre_hidden_t = batch_hidden.Slice(pre_h_start, pre_h_end); + math::matmul(device_ctx, pre_hidden_t, false, *weight, false, + static_cast(1.0), &gate_t, static_cast(1.0)); } // else if : FIXME support the initial hidden and cell @@ -112,27 +123,186 @@ class LSTMKernel : public framework::OpKernel { lstm_value.outputValue = out_t.data(); lstm_value.stateValue = cell_t.data(); lstm_value.stateActiveValue = cell_pre_act_t.data(); - math::LstmUnitFunctor::compute(ctx.device_context(), lstm_value, + math::LstmUnitFunctor::compute(device_ctx, lstm_value, frame_size, cur_batch_size, gate_act, cell_act, cand_act); lstm_value.prevStateValue = lstm_value.stateValue; } math::Batch2LoDTensorFunctor to_seq; - batch_out.set_lod(batch_gate->lod()); + batch_hidden.set_lod(batch_gate->lod()); // restore the output hidden in LoDTensor from the batch hidden - to_seq(ctx.device_context(), batch_out, *hidden_out); + to_seq(device_ctx, batch_hidden, *hidden_out); batch_cell.set_lod(batch_gate->lod()); // restore the output cell state in LoDTensor from the batch cell - to_seq(ctx.device_context(), batch_cell, *cell_out); + to_seq(device_ctx, batch_cell, *cell_out); } }; template class LSTMGradKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& ctx) const override {} + void Compute(const framework::ExecutionContext& ctx) const override { + auto* input = ctx.Input("Input"); + auto* weight = ctx.Input("Weight"); + auto* bias = ctx.Input("Bias"); + + auto* hidden_out = ctx.Input("Hidden"); + auto* cell_out = ctx.Input("Cell"); + + auto* batch_gate = ctx.Input("BatchGate"); + auto* batch_cell_pre_act = ctx.Input("BatchCellPreAct"); + + auto* hidden_g = ctx.Input(framework::GradVarName("Hidden")); + + auto* in_g = ctx.Output(framework::GradVarName("Input")); + auto* weight_g = ctx.Output(framework::GradVarName("Weight")); + auto* bias_g = ctx.Output(framework::GradVarName("Bias")); + + auto& device_ctx = ctx.device_context(); + math::SetConstant zero; + if (weight_g) { + weight_g->mutable_data(ctx.GetPlace()); + zero(device_ctx, weight_g, static_cast(0.0)); + } + + auto in_dims = input->dims(); + auto out_dims = hidden_g->dims(); + int frame_size = static_cast(in_dims[1] / 4); + PADDLE_ENFORCE_EQ(frame_size, out_dims[1]); + + math::LstmMetaValue lstm_value; + if (bias) { + T* bias_data = const_cast(bias->data()); + lstm_value.checkIg = bias_data + 4 * frame_size; + lstm_value.checkFg = lstm_value.checkIg + frame_size; + lstm_value.checkOg = lstm_value.checkFg + frame_size; + } else { + lstm_value.checkIg = nullptr; + lstm_value.checkFg = nullptr; + lstm_value.checkOg = nullptr; + } + + math::LstmMetaGrad lstm_grad; + if (bias && bias_g) { + T* bias_g_data = const_cast(bias_g->mutable_data(ctx.GetPlace())); + zero(device_ctx, bias_g, static_cast(0.0)); + lstm_grad.checkIgGrad = bias_g_data + 4 * frame_size; + lstm_grad.checkFgGrad = lstm_grad.checkIgGrad + frame_size; + lstm_grad.checkOgGrad = lstm_grad.checkFgGrad + frame_size; + } else { + lstm_grad.checkIgGrad = nullptr; + lstm_grad.checkFgGrad = nullptr; + lstm_grad.checkOgGrad = nullptr; + } + + math::LoDTensor2BatchFunctor to_batch; + + // use the local variable as here. + LoDTensor batch_hidden; + batch_hidden.mutable_data(out_dims, ctx.GetPlace()); + batch_hidden.set_lod(batch_gate->lod()); + to_batch(device_ctx, *hidden_out, batch_hidden, false); + + LoDTensor batch_hidden_g; + batch_hidden_g.mutable_data(out_dims, ctx.GetPlace()); + batch_hidden_g.set_lod(batch_gate->lod()); + to_batch(device_ctx, *hidden_g, batch_hidden_g, false); + + LoDTensor batch_cell; + batch_cell.mutable_data(out_dims, ctx.GetPlace()); + batch_cell.set_lod(batch_gate->lod()); + to_batch(device_ctx, *cell_out, batch_cell, false); + + LoDTensor batch_cell_g; + batch_cell_g.mutable_data(out_dims, ctx.GetPlace()); + batch_cell_g.set_lod(batch_gate->lod()); + // TODO(qingqing) support the case output cell has gradient. + // to_batch(device_ctx, *cell_g, batch_cell_g, false); + zero(device_ctx, &batch_cell_g, static_cast(0.0)); + + LoDTensor batch_gate_g; + batch_gate_g.mutable_data(batch_gate->dims(), ctx.GetPlace()); + batch_gate_g.set_lod(batch_gate->lod()); + + auto gate_act = ctx.Attr("gateActivation"); + auto cell_act = ctx.Attr("cellActivation"); + auto cand_act = ctx.Attr("candidateActivation"); + + auto batch_starts = batch_gate->lod()[0]; + size_t num_batch = batch_starts.size() - 1; + for (int n = static_cast(num_batch) - 1; n >= 0; n--) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + + Tensor gate = batch_gate->Slice(bstart, bend); + Tensor cell = batch_cell.Slice(bstart, bend); + Tensor cell_pre_act = batch_cell_pre_act->Slice(bstart, bend); + lstm_value.gateValue = gate.data(); + lstm_value.stateValue = cell.data(); + lstm_value.stateActiveValue = cell_pre_act.data(); + + Tensor out_g = batch_hidden_g.Slice(bstart, bend); + Tensor gate_g = batch_gate_g.Slice(bstart, bend); + Tensor cell_g = batch_cell_g.Slice(bstart, bend); + lstm_grad.stateGrad = cell_g.data(); + lstm_grad.gateGrad = gate_g.data(); + lstm_grad.outputGrad = out_g.data(); + + if (n) { + int bstart_pre = static_cast(batch_starts[n - 1]); + Tensor cell_pre = batch_cell.Slice(bstart_pre, bstart); + Tensor cell_pre_g = batch_cell_g.Slice(bstart_pre, bstart); + lstm_value.prevStateValue = cell_pre.data(); + lstm_grad.prevStateGrad = cell_pre_g.data(); + } else { + lstm_value.prevStateValue = nullptr; + lstm_grad.prevStateGrad = nullptr; + } + + int cur_batch_size = bend - bstart; + math::LstmUnitGradFunctor::compute( + device_ctx, lstm_value, lstm_grad, frame_size, cur_batch_size, + gate_act, cell_act, cand_act); + + if (n != 0) { + int pre_h_start = static_cast(batch_starts[n - 1]); + int pre_h_end = pre_h_start + cur_batch_size; + auto pre_hidden_g = batch_hidden_g.Slice(pre_h_start, pre_h_end); + math::matmul(device_ctx, gate_g, false, *weight, true, + static_cast(1.0), &pre_hidden_g, + static_cast(1.0)); + if (weight_g) { + /* backward weight */ + auto pre_hidden = batch_hidden.Slice(pre_h_start, pre_h_end); + math::matmul(device_ctx, pre_hidden, true, gate_g, false, + static_cast(1.0), weight_g, + static_cast(1.0)); + } + } + } + + math::Batch2LoDTensorFunctor to_seq; + if (in_g) { + /* backward data */ + in_g->mutable_data(ctx.GetPlace()); + to_seq(device_ctx, batch_gate_g, *in_g); + } + if (bias && bias_g) { + /* backward bias */ + int m = static_cast(batch_gate_g.dims()[0]); + int n = static_cast(batch_gate_g.dims()[1]); + + Tensor ones; + ones.mutable_data({m}, ctx.GetPlace()); + math::SetConstant set; + set(device_ctx, &ones, static_cast(1.0)); + + math::gemv(device_ctx, true, m, n, 1., batch_gate_g.data(), + ones.data(), 0., bias_g->data()); + } + } }; } // namespace operators diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc index 5d63017208a55ec4bcc2e8d66f1ca2e1b84d4593..f4519ec16f3f694cf49941f8d23c4106f6f1ddc3 100644 --- a/paddle/operators/lstm_unit_op.cc +++ b/paddle/operators/lstm_unit_op.cc @@ -57,17 +57,22 @@ class LstmUnitOpMaker : public framework::OpProtoAndCheckerMaker { "The cell state tensor of last time-step in the Lstm Unit operator."); AddOutput("C", "The cell tensor of Lstm Unit operator."); AddOutput("H", "The hidden state tensor of Lstm Unit operator."); - - AddComment(R"DOC(Lstm-Unit Operator + AddAttr("forget_bias", + "(float, default 0.0) " + "The forget bias of Lstm Unit.") + .SetDefault(0.0); + AddComment(R"DOC( +Lstm Unit Operator Equation: - i, f, o, j = split(X) - C = C_prev * sigm(f + forget_bias) + sigm(i) * tanh(j) - H = C * sigm(o) + +$$ +i, f, o, j = split(X) \\ +C = C_{prev} * sigm(f + forget\_bias) + sigm(i) * tanh(j) \\ +H = C * sigm(o) +$$ )DOC"); - AddAttr("forget_bias", "The forget bias of Lstm Unit.") - .SetDefault(0.0); } }; diff --git a/paddle/operators/lstm_unit_op.cu b/paddle/operators/lstm_unit_op.cu index 49ea550b6f49a13bf31d14321d7a9eb13a834d4b..e192283aa0afac49e8e467506f3703d1ce60d2a6 100644 --- a/paddle/operators/lstm_unit_op.cu +++ b/paddle/operators/lstm_unit_op.cu @@ -12,6 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* Acknowledgement: the following code is strongly inspired by +https://github.com/caffe2/caffe2/blob/master/caffe2/operators/lstm_unit_op_gpu.cu +*/ + #include "paddle/framework/op_registry.h" #include "paddle/operators/cross_entropy_op.h" #include "paddle/platform/assert.h" diff --git a/paddle/operators/lstm_unit_op.h b/paddle/operators/lstm_unit_op.h index 625b1852c2f0eb2ed435f73fea251c40c614a7dd..38cb298f92a21bb5c7508761fec701d28279a85f 100644 --- a/paddle/operators/lstm_unit_op.h +++ b/paddle/operators/lstm_unit_op.h @@ -12,6 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* Acknowledgement: the following code is strongly inspired by +https://github.com/caffe2/caffe2/blob/master/caffe2/operators/lstm_unit_op.h +*/ + #pragma once #include "glog/logging.h" #include "paddle/framework/op_registry.h" diff --git a/paddle/operators/margin_rank_loss_op.cc b/paddle/operators/margin_rank_loss_op.cc index 638a99addc2119e8f44648cc54b97bd8a892d2bc..d7e8a0ea7632650203106b01531d724cf0b8e085 100644 --- a/paddle/operators/margin_rank_loss_op.cc +++ b/paddle/operators/margin_rank_loss_op.cc @@ -55,8 +55,6 @@ class MarginRankLossOpMaker : public framework::OpProtoAndCheckerMaker { "(2-D tensor with shape [batch_size x 1]) " "The label indicating X1 ranked higher than X2 or not, " "can only be +1 or -1."); - AddAttr("margin", "(scalar, default 0) Margin for MarginRankLossOp.") - .SetDefault(static_cast(0)); AddOutput("Activated", "(2-D tensor with shape [batch_size x 1]) Intermediate tensor " "to indicate whether each element of Output(Out) is activated.") @@ -64,23 +62,26 @@ class MarginRankLossOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(2-D tensor with shape [batch_size x 1]) " "The output loss of MarginRankLoss operator."); + AddAttr("margin", "(scalar, default 0) Margin for MarginRankLossOp.") + .SetDefault(static_cast(0)); AddComment(R"DOC( +MarginRankLoss Operator. -MarginRankLoss operator measures the loss given a pair of training sample +This operator measures the loss given a pair of training sample {`X1`, `X2`} and the `Label` with attribute `margin`, where `Label = +1` -indicating X1 is ranked higher than `X2`, otherwise `Label = -1`. The loss -turns out +indicating X1 is ranked higher than `X2` and `Label = -1` otherwise. The loss +is calculated as: -loss(X1, X2, Label) = max(0, -Label * (X1 - X2) + margin). +$loss(X1, X2, Label) = \max(0, -Label * (X1 - X2) + margin)$ -The attribute `margin` involved here helps make the predictions more robust. +The attribute `margin` here helps make the predictions more robust. Denote the item ranked higher as the positive sample, otherwise the negative sample. If the score of the two samples satisfies -positive sample - negative sample < margin, +$positive sample - negative sample < margin$ -the pair of samples will contribute to the final loss, which will backpropogate -and train the ranking model to enlarge the difference of the two score. +the pair of samples will contribute to the final loss, which will backpropagate +and train the ranking model to enlarge the difference between the two scores. For batch input with size `batch_size`, `X1`, `X2` and `Label` all have the same shape [batch_size x 1]. diff --git a/paddle/operators/math/context_project.h b/paddle/operators/math/context_project.h index e37f3a5bf2bd59e46f66aa3a8284e05d79dbc790..e0283360414fbdfb3dae2e94b45c9c8daeed3c74 100644 --- a/paddle/operators/math/context_project.h +++ b/paddle/operators/math/context_project.h @@ -16,36 +16,36 @@ limitations under the License. */ #include "paddle/framework/eigen.h" #include "paddle/framework/lod_tensor.h" -#include "paddle/framework/tensor.h" #include "paddle/operators/math/im2col.h" namespace paddle { namespace operators { namespace math { +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; template using EigenMatrix = framework::EigenMatrix; + /* - * \brief Context projection concatenate features in adjacent time steps in + * \brief Context projection concatenates features in adjacent time-steps in * a sequence. The i-th row of the output is the concatenation of * context_length rows of the input. The context_length rows are the * consecutive rows from the i+shift_start row. - + * ContextProjectGradFunctor is the inverse process of ContextProjectFunctor. + * * \param in Input data. - * \param Shape The shape of Input data, - * [minibatch, number_of_input_features]. - * \param type A float LoDTensor. + * \param Shape The shape of Input data: + * [mini-batch, input_hidden_size]. * * \param padding_data Padding data. - * \param Shape The shape of Padding data, - * [up_pad + down_pad, number_of_input_features]. - * \param type A float Tensor. + * \param Shape The shape of Padding data: + * [up_pad + down_pad, input_hidden_size]. * * \param col Col data. - * \param Shape The shape of Col data, - * [minibatch, context_length * number_of_input_features]. - * \param type A float Tensor. + * \param Shape The shape of Col data: + * [mini-batch, context_length * input_hidden_size]. * * For a mini-batch of 2 variable lengths sentences, containing 3, and 1 * time-steps: @@ -63,72 +63,170 @@ using EigenMatrix = framework::EigenMatrix; * representation is 2. * * - Case1: - * If context_start is -1 and padding_trainable is false, we use zero to pad - * instead of learned weight to pad, - * and the context_lenth is 3, the output (Out) is: + * If context_start is -1 and padding_trainable is false, we use zero to pad + * instead of learned weight to pad, + * and the context_length is 3, the output (Out) is: * - * Out =[[0, 0, a1, a2, b1, b2; - * a1, a2, b1, b2, c1, c2; - * b1, b2, c1, c2, 0, 0 ] - * [0, 0, d1, d2, 0, 0 ]] + * Out =[[0, 0, a1, a2, b1, b2; + * a1, a2, b1, b2, c1, c2; + * b1, b2, c1, c2, 0, 0 ] + * [0, 0, d1, d2, 0, 0 ]] * * - Case2: - * If context_start is -1 and padding_trainable is true, we use learned weight - * to pad, - * and the context_lenth is 3, the output (Out) is: + * If context_start is -1 and padding_trainable is true, we use learned weight + * to pad, + * and the context_length is 3, the output (Out) is: * - * Out = [[w1, w2, a1, a2, b1, b2; - * a1, a2, b1, b2, c1, c2; - * b1, b2, c1, c2, w3, w4] - * [w1, w2, d1, d2, w3, w4]] + * Out = [[w1, w2, a1, a2, b1, b2; + * a1, a2, b1, b2, c1, c2; + * b1, b2, c1, c2, w3, w4] + * [w1, w2, d1, d2, w3, w4]] * */ template class ContextProjectFunctor { public: - void operator()(const platform::DeviceContext& context, - framework::LoDTensor& in, framework::Tensor& padding_data, - framework::Tensor& col, bool padding_trainable, + void operator()(const platform::DeviceContext& context, const LoDTensor& in, + const Tensor& padding_data, Tensor& col, + bool padding_trainable, int context_start, int context_length, + int context_stride, int up_pad, int down_pad) { + auto lod_level_0 = in.lod()[0]; + + math::Im2ColFunctor im2col_ocf; + + int input_row_begin, input_row_end; + int sequence_height, sequence_width; + sequence_width = in.dims()[1]; + + for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { + input_row_begin = (context_start > 0) + ? static_cast(lod_level_0[i]) + context_start + : static_cast(lod_level_0[i]); + input_row_end = static_cast(lod_level_0[i + 1]); + + Tensor out_t = col.Slice(static_cast(lod_level_0[i]), + static_cast(lod_level_0[i + 1])); + + sequence_height = static_cast(out_t.dims()[0]); + + if (input_row_begin < input_row_end) { + Tensor in_t = in.Slice(input_row_begin, input_row_end); + + std::vector output_shape( + {sequence_height, 1, 1, context_length, + sequence_width}); // output_height, output_width, + // input_channels, filter_height, filter_width + out_t.Resize(framework::make_ddim(output_shape)); + + std::vector input_shape( + {1, input_row_end - input_row_begin, + sequence_width}); // input_channels, input_height, input_width + in_t.Resize(framework::make_ddim(input_shape)); + + im2col_ocf(context, in_t, out_t, + /*stride_height*/ context_stride, /*stride_width*/ 1, up_pad, + down_pad, 0, 0); + out_t.Resize({sequence_height, context_length * sequence_width}); + } + } + if (padding_trainable) { + for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { + Tensor out_t = col.Slice(static_cast(lod_level_0[i]), + static_cast(lod_level_0[i + 1])); + + sequence_height = static_cast(out_t.dims()[0]); + + // add up trainable data + out_t.Resize({sequence_height * context_length, sequence_width}); + + if (up_pad > 0) { // add up pad + int padding_rows = std::min( + up_pad, static_cast(lod_level_0[i + 1] - lod_level_0[i])); + + for (int k = 0; k < padding_rows; ++k) { + int padding_size = + k + context_length < up_pad ? context_length : up_pad - k; + Tensor out_t_sub = out_t.Slice(k * context_length, + k * context_length + padding_size); + Tensor w_sub = padding_data.Slice(k, k + padding_size); + auto out_t_sub_e = EigenMatrix::From(out_t_sub); + auto w_sub_e = EigenMatrix::From(w_sub); + out_t_sub_e.device(*context.GetEigenDevice()) = w_sub_e; + } + } + if (down_pad > 0) { // add down pad + int down_pad_begin_row = + std::max(0, + (sequence_height - context_start - context_length) + 1) + + 1; + int padding_begin = std::max(0, context_start - sequence_height); + int padding_size = + sequence_height - context_start >= context_length + ? 1 + : context_length - (sequence_height - context_start); + if (context_start >= sequence_height) padding_size = context_length; + int padding_idx = padding_begin; + for (int t = 0; t + down_pad_begin_row <= sequence_height; + ++t, ++padding_size) { + if (context_start >= sequence_height) padding_size = context_length; + if (padding_size > context_length) { + padding_size = context_length; + padding_idx++; + } + if (padding_begin > 0 || sequence_height == context_start) + padding_idx = padding_begin + t; + + Tensor out_t_sub = out_t.Slice( + (down_pad_begin_row + t) * context_length - padding_size, + (down_pad_begin_row + t) * context_length); + Tensor w_sub = padding_data.Slice( + up_pad + padding_idx, up_pad + padding_idx + padding_size); + auto out_t_sub_e = EigenMatrix::From(out_t_sub); + auto w_sub_e = EigenMatrix::From(w_sub); + out_t_sub_e.device(*context.GetEigenDevice()) = w_sub_e; + } + } + out_t.Resize({sequence_height, context_length * sequence_width}); + } + } + } +}; + +template +class ContextProjectGradFunctor { + public: + void operator()(const platform::DeviceContext& context, LoDTensor& in, + Tensor& padding_data, Tensor& col, bool padding_trainable, int context_start, int context_length, int context_stride, - int up_pad, int down_pad, bool gradient, bool input_grad, - bool pad_grad) { + int up_pad, int down_pad, bool input_grad, bool pad_grad) { auto lod_level_0 = in.lod()[0]; - paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kOCF, Place, float> - im2col_ocf; - paddle::operators::math::Col2ImFunctor< - paddle::operators::math::ColFormat::kOCF, Place, float> - col2im_ocf; + math::Col2ImFunctor col2im_ocf; int input_row_begin, input_row_end; int sequence_height, sequence_width; sequence_width = in.dims()[1]; - input_grad = gradient && input_grad; - pad_grad = gradient && pad_grad; - if (!gradient || input_grad) { + if (input_grad) { for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { input_row_begin = (context_start > 0) ? static_cast(lod_level_0[i]) + context_start : static_cast(lod_level_0[i]); input_row_end = static_cast(lod_level_0[i + 1]); - framework::Tensor out_t = - col.Slice(static_cast(lod_level_0[i]), - static_cast(lod_level_0[i + 1])); + Tensor out_t = col.Slice(static_cast(lod_level_0[i]), + static_cast(lod_level_0[i + 1])); sequence_height = static_cast(out_t.dims()[0]); if (input_row_begin < input_row_end) { - framework::Tensor in_t = in.Slice(input_row_begin, input_row_end); + Tensor in_t = in.Slice(input_row_begin, input_row_end); std::vector output_shape( {sequence_height, 1, 1, context_length, sequence_width}); // output_height, output_width, // input_channels, filter_height, filter_width - out_t.Resize(framework::make_ddim(output_shape)); std::vector input_shape( @@ -136,53 +234,39 @@ class ContextProjectFunctor { sequence_width}); // input_channels, input_height, input_width in_t.Resize(framework::make_ddim(input_shape)); - if (gradient) { - col2im_ocf(context, in_t, out_t, - /*stride_height*/ context_stride, /*stride_width*/ 1, - up_pad, down_pad, 0, 0); - } else { - im2col_ocf(context, in_t, out_t, - /*stride_height*/ context_stride, /*stride_width*/ 1, - up_pad, down_pad, 0, 0); - } + col2im_ocf(context, in_t, out_t, + /*stride_height*/ context_stride, /*stride_width*/ 1, + up_pad, down_pad, 0, 0); out_t.Resize({sequence_height, context_length * sequence_width}); } } } - if (!gradient || pad_grad) { + if (pad_grad) { if (padding_trainable) { for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { - framework::Tensor out_t = - col.Slice(static_cast(lod_level_0[i]), - static_cast(lod_level_0[i + 1])); + Tensor out_t = col.Slice(static_cast(lod_level_0[i]), + static_cast(lod_level_0[i + 1])); sequence_height = static_cast(out_t.dims()[0]); - - // add up trainable data out_t.Resize({sequence_height * context_length, sequence_width}); - if (up_pad > 0) { // add up pad + if (up_pad > 0) { int padding_rows = std::min( up_pad, static_cast(lod_level_0[i + 1] - lod_level_0[i])); for (int k = 0; k < padding_rows; ++k) { int padding_size = k + context_length < up_pad ? context_length : up_pad - k; - framework::Tensor out_t_sub = out_t.Slice( - k * context_length, k * context_length + padding_size); - framework::Tensor w_sub = padding_data.Slice(k, k + padding_size); - // in this block, using EigenVector::Flatten is ok too. + Tensor out_t_sub = out_t.Slice(k * context_length, + k * context_length + padding_size); + Tensor w_sub = padding_data.Slice(k, k + padding_size); auto out_t_sub_e = EigenMatrix::From(out_t_sub); auto w_sub_e = EigenMatrix::From(w_sub); - if (gradient) { - w_sub_e.device(*context.GetEigenDevice()) = - w_sub_e + out_t_sub_e; - } else { - out_t_sub_e.device(*context.GetEigenDevice()) = w_sub_e; - } + w_sub_e.device(*context.GetEigenDevice()) = + w_sub_e + out_t_sub_e; } } - if (down_pad > 0) { // add down pad + if (down_pad > 0) { int down_pad_begin_row = std::max( 0, (sequence_height - context_start - context_length) + 1) + @@ -204,19 +288,16 @@ class ContextProjectFunctor { } if (padding_begin > 0 || sequence_height == context_start) padding_idx = padding_begin + t; - framework::Tensor out_t_sub = out_t.Slice( + + Tensor out_t_sub = out_t.Slice( (down_pad_begin_row + t) * context_length - padding_size, (down_pad_begin_row + t) * context_length); - framework::Tensor w_sub = padding_data.Slice( + Tensor w_sub = padding_data.Slice( up_pad + padding_idx, up_pad + padding_idx + padding_size); auto out_t_sub_e = EigenMatrix::From(out_t_sub); auto w_sub_e = EigenMatrix::From(w_sub); - if (gradient) { - w_sub_e.device(*context.GetEigenDevice()) = - w_sub_e + out_t_sub_e; - } else { - out_t_sub_e.device(*context.GetEigenDevice()) = w_sub_e; - } + w_sub_e.device(*context.GetEigenDevice()) = + w_sub_e + out_t_sub_e; } } out_t.Resize({sequence_height, context_length * sequence_width}); diff --git a/paddle/operators/math/detail/CMakeLists.txt b/paddle/operators/math/detail/CMakeLists.txt index 49cf228de2204cb4888cf645a0cb68ed04cc3371..92eac9d3623ceb5464133b5e7baa2e30f764805f 100644 --- a/paddle/operators/math/detail/CMakeLists.txt +++ b/paddle/operators/math/detail/CMakeLists.txt @@ -1,5 +1,3 @@ if(WITH_AVX) - cc_library(activation_functions SRCS hl_cpu_functions.cc hl_avx_functions.cc) -else() - cc_library(activation_functions SRCS hl_cpu_functions.cc) + cc_library(activation_functions SRCS avx_functions.cc) endif() diff --git a/paddle/operators/math/detail/activation_functions.h b/paddle/operators/math/detail/activation_functions.h new file mode 100644 index 0000000000000000000000000000000000000000..a20c35d1d9dc4a3a6fae92023fd1aae787a716ec --- /dev/null +++ b/paddle/operators/math/detail/activation_functions.h @@ -0,0 +1,170 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/platform/hostdevice.h" + +#ifdef __AVX__ +#include +#endif + +namespace paddle { +namespace operators { +namespace math { +namespace detail { + +#define SIGMOID_THRESHOLD_MIN -40.0 +#define SIGMOID_THRESHOLD_MAX 13.0 +#define EXP_MAX_INPUT 40.0 + +namespace forward { + +template +DEVICE T Identity(const T a) { + return a; +} + +template +DEVICE T Relu(const T a) { + return a > static_cast(0.0) ? a : static_cast(0.0); +} + +template +DEVICE T Sigmoid(const T a) { + const T min = SIGMOID_THRESHOLD_MIN; + const T max = SIGMOID_THRESHOLD_MAX; + T tmp = (a < min) ? min : ((a > max) ? max : a); + return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); +} + +template +DEVICE T Tanh(const T a) { + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + return (2.0 / (1.0 + exp(tmp))) - 1.0; +} + +} // namespace forward + +namespace backward { + +template +DEVICE T Identity(const T a, const T b) { + return a; +} + +template +DEVICE T Relu(const T a, const T b) { + return a * (b > 0.0 ? 1.0 : 0.0); +} + +template +DEVICE T Sigmoid(const T a, const T b) { + return a * b * (1.0 - b); +} + +template +DEVICE T Tanh(const T a, const T b) { + return a * (1.0 - b * b); +} + +} // namespace backward + +template +struct Active { + typedef T (*Act)(T); + typedef T (*ActGrad)(T, T); +}; + +static DEVICE Active::Act kActFloat[] = { + &forward::Sigmoid, &forward::Relu, &forward::Tanh, + &forward::Identity}; + +static DEVICE Active::ActGrad kActGradFloat[] = { + &backward::Sigmoid, &backward::Relu, &backward::Tanh, + &backward::Identity}; + +static DEVICE Active::Act kActDouble[] = { + &forward::Sigmoid, &forward::Relu, &forward::Tanh, + &forward::Identity}; + +static DEVICE Active::ActGrad kActGradDouble[] = { + &backward::Sigmoid, &backward::Relu, + &backward::Tanh, &backward::Identity}; + +namespace forward { +inline DEVICE float activation(float a, int index) { + return kActFloat[index](a); +} + +inline DEVICE double activation(double a, int index) { + return kActDouble[index](a); +} + +} // namespace forward + +namespace backward { +inline DEVICE float activation(float a, float b, int index) { + return kActGradFloat[index](a, b); +} + +inline DEVICE double activation(double a, double b, int index) { + return kActGradDouble[index](a, b); +} +} // namespace backward + +#ifdef __AVX__ +namespace forward { +namespace avx { +__m256 Relu(const __m256 a); +__m256 Sigmoid(const __m256 a); +__m256 Tanh(const __m256 a); +__m256 Identity(const __m256 a); +} // namespace avx +} // namespace forward + +namespace backward { +namespace avx { +__m256 Relu(const __m256 a, const __m256 b); +__m256 Sigmoid(const __m256 a, const __m256 b); +__m256 Tanh(const __m256 a, const __m256 b); +__m256 Identity(const __m256 a, const __m256 b); +} // namespace avx +} // namespace backward + +static Active<__m256>::Act kActAvx[] = { + &forward::avx::Sigmoid, &forward::avx::Relu, &forward::avx::Tanh, + &forward::avx::Identity}; + +static Active<__m256>::ActGrad kActGradAvx[] = { + &backward::avx::Sigmoid, &backward::avx::Relu, &backward::avx::Tanh, + &backward::avx::Identity}; + +namespace forward { +inline __m256 activation(__m256 a, int index) { return kActAvx[index](a); } +} // namespace forward + +namespace backward { +inline __m256 activation(__m256 a, __m256 b, int index) { + return kActGradAvx[index](a, b); +} +} // namespace backward + +#endif + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/hl_avx_functions.cc b/paddle/operators/math/detail/avx_functions.cc similarity index 68% rename from paddle/operators/math/detail/hl_avx_functions.cc rename to paddle/operators/math/detail/avx_functions.cc index 415bac5d93ee00244d072b0998c6941b14d4f8d8..6d9df654a48e990ec54d59c1e627aa1304122b21 100644 --- a/paddle/operators/math/detail/hl_avx_functions.cc +++ b/paddle/operators/math/detail/avx_functions.cc @@ -13,58 +13,74 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include "hl_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" // TODO(qingqing) refine this dependence #include "paddle/cuda/src/avx_mathfun.h" -namespace hppl { +namespace paddle { +namespace operators { +namespace math { +namespace detail { -__m256 exp(__m256 a) { return exp256_ps(a); } +__m256 Exp(__m256 a) { return exp256_ps(a); } -__m256 relu(const __m256 a) { +namespace forward { +namespace avx { +__m256 Relu(const __m256 a) { __m256 tmp = _mm256_set1_ps(0.0f); return _mm256_max_ps(a, tmp); } -__m256 sigmoid(const __m256 a) { +__m256 Sigmoid(const __m256 a) { __m256 max = _mm256_set1_ps(SIGMOID_THRESHOLD_MAX); __m256 min = _mm256_set1_ps(SIGMOID_THRESHOLD_MIN); __m256 tmp = _mm256_max_ps(a, min); tmp = _mm256_min_ps(tmp, max); tmp = _mm256_sub_ps(_mm256_set1_ps(0.0f), tmp); - tmp = exp(tmp); + tmp = Exp(tmp); tmp = _mm256_add_ps(_mm256_set1_ps(1.0f), tmp); tmp = _mm256_div_ps(_mm256_set1_ps(1.0f), tmp); return tmp; } -__m256 tanh(const __m256 a) { +__m256 Tanh(const __m256 a) { __m256 max = _mm256_set1_ps(EXP_MAX_INPUT); __m256 tmp = _mm256_mul_ps(_mm256_set1_ps(-2.0f), a); tmp = _mm256_min_ps(tmp, max); - tmp = exp(tmp); + tmp = Exp(tmp); return _mm256_sub_ps(_mm256_div_ps(_mm256_set1_ps(2.0f), _mm256_add_ps(_mm256_set1_ps(1.0f), tmp)), _mm256_set1_ps(1.0f)); } -__m256 linear(const __m256 a) { return a; } +__m256 Identity(const __m256 a) { return a; } -__m256 relu(const __m256 a, const __m256 b) { +} // namespace avx +} // namespace forward + +namespace backward { +namespace avx { +__m256 Relu(const __m256 a, const __m256 b) { return _mm256_mul_ps( a, _mm256_and_ps(_mm256_cmp_ps(b, _mm256_set1_ps(0.0f), _CMP_GT_OS), _mm256_set1_ps(1.0f))); } -__m256 sigmoid(const __m256 a, const __m256 b) { +__m256 Sigmoid(const __m256 a, const __m256 b) { return _mm256_mul_ps(_mm256_mul_ps(a, b), _mm256_sub_ps(_mm256_set1_ps(1.0f), b)); } -__m256 tanh(const __m256 a, const __m256 b) { +__m256 Tanh(const __m256 a, const __m256 b) { return _mm256_mul_ps( a, _mm256_sub_ps(_mm256_set1_ps(1.0f), _mm256_mul_ps(b, b))); } -__m256 linear(const __m256 a, const __m256 b) { return a; } -} // namespace hppl +__m256 Identity(const __m256 a, const __m256 b) { return a; } +} // namespace avx +} // namespace backward + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/hl_activation_functions.h b/paddle/operators/math/detail/hl_activation_functions.h deleted file mode 100644 index 9d7d9914f0090bff17049038dfa2288d84f3dbda..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_activation_functions.h +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_ACTIVATION_FUNCTIONS_H_ -#define HL_ACTIVATION_FUNCTIONS_H_ - -#include "hl_functions.h" -#include "paddle/operators/math/lstm_compute.h" - -/** - * Active functions: sigmoid, relu, tanh and linear. - */ -#define FLOAT_ACTIVE_FUNCTION \ - { \ - hppl::typef::sigmoid, hppl::typef::relu, hppl::typef::tanh, \ - hppl::typef::linear \ - } - -#define DOUBLE_ACTIVE_FUNCTION \ - { \ - hppl::typed::sigmoid, hppl::typed::relu, hppl::typed::tanh, \ - hppl::typed::linear \ - } - -#define AVX_ACTIVE_FUNCTION \ - { hppl::sigmoid, hppl::relu, hppl::tanh, hppl::linear } - -namespace hppl { - -using activation_mode_t = paddle::operators::math::activation_mode_t; - -/** - * Hppl supports sigmoid, relu, tanh, linear active functions - * for neural networks' forward and backward activation. - */ -template -class Active { - public: - typedef T (*forward)(T); - typedef T (*backward)(T, T); -}; - -template -struct ForwardActType; - -template <> -struct ForwardActType { - using type = Active::forward; -}; - -template <> -struct ForwardActType { - using type = Active::forward; -}; - -template -struct BackwardActType; - -template <> -struct BackwardActType { - using type = Active::backward; -}; - -template <> -struct BackwardActType { - using type = Active::backward; -}; - -#ifdef __NVCC__ -namespace gpu { -static __device__ Active::forward forward[] = FLOAT_ACTIVE_FUNCTION; -static __device__ Active::backward backward[] = FLOAT_ACTIVE_FUNCTION; - -static __device__ Active::forward forward_d[] = DOUBLE_ACTIVE_FUNCTION; -static __device__ Active::backward backward_d[] = - DOUBLE_ACTIVE_FUNCTION; - -template -struct ForwardAct { - __device__ typename ForwardActType::type operator()( - activation_mode_t type); -}; - -template <> -struct ForwardAct { - __device__ ForwardActType::type operator()(activation_mode_t type) { - return forward[type]; - } -}; - -template <> -struct ForwardAct { - __device__ ForwardActType::type operator()(activation_mode_t type) { - return forward_d[type]; - } -}; - -template -struct BackwardAct { - __device__ typename BackwardActType::type operator()( - activation_mode_t type); -}; - -template <> -struct BackwardAct { - __device__ BackwardActType::type operator()(activation_mode_t type) { - return backward[type]; - } -}; - -template <> -struct BackwardAct { - __device__ BackwardActType::type operator()(activation_mode_t type) { - return backward_d[type]; - } -}; - -} // namespace gpu -#else -namespace cpu { -static Active::forward forward[] = FLOAT_ACTIVE_FUNCTION; -static Active::backward backward[] = FLOAT_ACTIVE_FUNCTION; - -static Active::forward forward_d[] = DOUBLE_ACTIVE_FUNCTION; -static Active::backward backward_d[] = DOUBLE_ACTIVE_FUNCTION; - -template -struct ForwardAct { - typename ForwardActType::type operator()(activation_mode_t type); -}; - -template <> -struct ForwardAct { - ForwardActType::type operator()(activation_mode_t type) { - return forward[type]; - } -}; - -template <> -struct ForwardAct { - ForwardActType::type operator()(activation_mode_t type) { - return forward_d[type]; - } -}; - -template -struct BackwardAct { - typename BackwardActType::type operator()(activation_mode_t type); -}; - -template <> -struct BackwardAct { - BackwardActType::type operator()(activation_mode_t type) { - return backward[type]; - } -}; - -template <> -struct BackwardAct { - BackwardActType::type operator()(activation_mode_t type) { - return backward_d[type]; - } -}; - -} // namespace cpu - -#ifdef __AVX__ -namespace avx { -static Active<__m256>::forward forward[] = AVX_ACTIVE_FUNCTION; -static Active<__m256>::backward backward[] = AVX_ACTIVE_FUNCTION; -} // namespace avx -#endif -#endif - -} // namespace hppl - -#endif // HL_ACTIVATION_FUNCTIONS_H_ diff --git a/paddle/operators/math/detail/hl_avx_functions.h b/paddle/operators/math/detail/hl_avx_functions.h deleted file mode 100644 index 35f4eabb4c07c6cc9d2edded02e5b6290b1232f8..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_avx_functions.h +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_AVX_FUNCTIONS_H_ -#define HL_AVX_FUNCTIONS_H_ - -#include - -namespace hppl { -__m256 relu(const __m256 a); -__m256 sigmoid(const __m256 a); -__m256 tanh(const __m256 a); -__m256 linear(const __m256 a); - -__m256 relu(const __m256 a, const __m256 b); -__m256 sigmoid(const __m256 a, const __m256 b); -__m256 tanh(const __m256 a, const __m256 b); -__m256 linear(const __m256 a, const __m256 b); -} // namespace hppl - -#endif // HL_AVX_FUNCTIONS_H_ diff --git a/paddle/operators/math/detail/hl_cpu_functions.cc b/paddle/operators/math/detail/hl_cpu_functions.cc deleted file mode 100644 index 21ec78f9629af0e4673a56517d76ac6734f57db8..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_cpu_functions.cc +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include -#include "hl_functions.h" - -namespace hppl { -namespace typef { - -float relu(const float a) { - return a > static_cast(0.0) ? a : static_cast(0.0); -} - -float sigmoid(const float a) { - const float min = SIGMOID_THRESHOLD_MIN; - const float max = SIGMOID_THRESHOLD_MAX; - float tmp = (a < min) ? min : ((a > max) ? max : a); - return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); -} - -float tanh(const float a) { - float tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(tmp))) - 1.0; -} - -float linear(const float a) { return a; } - -float relu(const float a, const float b) { return a * (b > 0.0 ? 1.0 : 0.0); } - -float sigmoid(const float a, const float b) { - return a * b * (static_cast(1) - b); -} - -float tanh(const float a, const float b) { - return a * (static_cast(1) - b * b); -} - -float linear(const float a, const float b) { return a; } - -} // namespace typef - -namespace typed { -double relu(const double a) { - return a > static_cast(0.0) ? a : static_cast(0.0); -} - -double sigmoid(const double a) { - const double min = SIGMOID_THRESHOLD_MIN; - const double max = SIGMOID_THRESHOLD_MAX; - double tmp = (a < min) ? min : ((a > max) ? max : a); - return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); -} - -double tanh(const double a) { - double tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(tmp))) - 1.0; -} - -double linear(const double a) { return a; } - -double relu(const double a, const double b) { - return a * (b > 0.0 ? 1.0 : 0.0); -} - -double sigmoid(const double a, const double b) { - return a * b * (static_cast(1) - b); -} - -double tanh(const double a, const double b) { - return a * (static_cast(1) - b * b); -} - -double linear(const double a, const double b) { return a; } - -} // namespace typed -} // namespace hppl diff --git a/paddle/operators/math/detail/hl_functions.h b/paddle/operators/math/detail/hl_functions.h deleted file mode 100644 index 3e2f0c9ee6d3ae2ed598c4d5f09b85b7d61fdd51..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_functions.h +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_FUNCTIONS_H_ -#define HL_FUNCTIONS_H_ - -/** - * sigmoid threshold maximum - */ -#define SIGMOID_THRESHOLD_MIN -40.0 - -/** - * sigmoid threshold minimum - */ -#define SIGMOID_THRESHOLD_MAX 13.0 - -/** - * The maximum input value for exp, used to avoid overflow problem. - * currently only used for tanh function. - */ -#define EXP_MAX_INPUT 40.0 - -#ifndef __NVCC__ -namespace hppl { -namespace typef { -float relu(const float a); -float sigmoid(const float a); -float tanh(const float a); -float linear(const float a); - -float relu(const float a, const float b); -float sigmoid(const float a, const float b); -float tanh(const float a, const float b); -float linear(const float a, const float b); - -} // namespace typef - -namespace typed { -double relu(const double a); -double sigmoid(const double a); -double tanh(const double a); -double linear(const double a); - -double relu(const double a, const double b); -double sigmoid(const double a, const double b); -double tanh(const double a, const double b); -double linear(const double a, const double b); -} // namespace typed - -} // namespace hppl - -#ifdef __AVX__ -#include "hl_avx_functions.h" -#endif - -#else -#include "hl_gpu_functions.h" -#endif - -#endif // HL_FUNCTIONS_H_ diff --git a/paddle/operators/math/detail/hl_gpu_functions.h b/paddle/operators/math/detail/hl_gpu_functions.h deleted file mode 100644 index 72f2204e7b2cfdba1367b51e3731dde11fb292d6..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_gpu_functions.h +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_GPU_FUNCTIONS_CUH_ -#define HL_GPU_FUNCTIONS_CUH_ - -#include "hl_base.h" - -namespace hppl { -namespace typef { - -__device__ static float relu(const float a) { return a > 0.0f ? a : 0.0f; } - -__device__ static float sigmoid(const float a) { - const float min = SIGMOID_THRESHOLD_MIN; - const float max = SIGMOID_THRESHOLD_MAX; - float tmp = (a < min) ? min : ((a > max) ? max : a); - return __fdividef(1.0f, 1.0f + __expf(-tmp)); -} - -__device__ static float tanh(const float a) { - float tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return __fdividef(2.0f, (1.0f + __expf(-2.0f * tmp))) - 1.0f; -} - -__device__ static float linear(const float a) { return a; } - -__device__ static float relu(const float a, const float b) { - return a * (b > 0.0f ? 1.0f : 0.0f); -} - -__device__ static float sigmoid(const float a, const float b) { - return a * b * (1.0f - b); -} - -__device__ static float tanh(const float a, const float b) { - return a * (1.0f - b * b); -} - -__device__ static float linear(const float a, const float b) { return a; } - -} // namespace typef - -namespace typed { - -__device__ static double relu(const double a) { return a > 0.0 ? a : 0.0; } - -__device__ static double sigmoid(const double a) { - const double min = SIGMOID_THRESHOLD_MIN; - const double max = SIGMOID_THRESHOLD_MAX; - double tmp = (a < min) ? min : ((a > max) ? max : a); - return 1.0 / (1.0 + exp(-tmp)); -} - -__device__ static double tanh(const double a) { - double tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(-2.0 * a))) - 1.0; -} - -__device__ static double linear(const double a) { return a; } - -__device__ static double relu(const double a, const double b) { - return a * (b > 0.0 ? 1.0 : 0.0); -} - -__device__ static double sigmoid(const double a, const double b) { - return a * b * (1 - b); -} - -__device__ static double tanh(const double a, const double b) { - return a * (1.0 - b * b); -} - -__device__ static double linear(const double a, const double b) { return a; } - -} // namespace typef - -} // namespace hppl - -#endif // HL_GPU_FUNCTIONS_CUH_ diff --git a/paddle/operators/math/detail/lstm_cpu_kernel.h b/paddle/operators/math/detail/lstm_cpu_kernel.h index 74d51d7bc9b91f4c8088384d77183131f57aafab..f5b0dd85c9d63805459431f933176581ee3658dc 100644 --- a/paddle/operators/math/detail/lstm_cpu_kernel.h +++ b/paddle/operators/math/detail/lstm_cpu_kernel.h @@ -14,7 +14,7 @@ limitations under the License. */ #pragma once #include -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/operators/math/lstm_compute.h" namespace paddle { @@ -60,10 +60,8 @@ void naive_lstm_forward_one_sequence(Op op, LstmMetaValue value, rPrevState = value.prevStateValue[i]; } - hppl::cpu::ForwardAct act; op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO, act(active_node), act(active_gate), - act(active_state)); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); valueIn[i] = rValueIn; valueIg[i] = rValueIg; @@ -127,11 +125,10 @@ void naive_lstm_backward_one_sequence(Op op, LstmMetaValue value, rPrevState = value.prevStateValue[i]; } - hppl::cpu::BackwardAct act; op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, - rCheckOGrad, act(active_node), act(active_gate), act(active_state)); + rCheckOGrad, active_node, active_gate, active_state); gradIn[i] = rGradIn; gradIg[i] = rGradIg; @@ -185,8 +182,7 @@ void avx_lstm_forward_one_sequence(Op op, LstmMetaValue value, int frameSize, } op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO, hppl::avx::forward[active_node], - hppl::avx::forward[active_gate], hppl::avx::forward[active_state]); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); valueIn[i] = rValueIn; valueIg[i] = rValueIg; @@ -255,8 +251,7 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, - rCheckOGrad, hppl::avx::backward[active_node], - hppl::avx::backward[active_gate], hppl::avx::backward[active_state]); + rCheckOGrad, active_node, active_gate, active_state); gradIn[i] = rGradIn; gradIg[i] = rGradIg; diff --git a/paddle/operators/math/detail/lstm_gpu_kernel.h b/paddle/operators/math/detail/lstm_gpu_kernel.h index 9573eaefb6a9d678ef70f2e2bffdc6a3011b21ea..41a54a359daa14a047c49728962ea15eefd12274 100644 --- a/paddle/operators/math/detail/lstm_gpu_kernel.h +++ b/paddle/operators/math/detail/lstm_gpu_kernel.h @@ -13,13 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/operators/math/lstm_compute.h" #include "paddle/platform/cuda_helper.h" #include "paddle/platform/device_context.h" -#include +#include namespace paddle { namespace operators { @@ -70,10 +69,8 @@ __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, rPrevState = value.prevStateValue[frameIdx]; } - hppl::gpu::ForwardAct act; op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO, act(active_node), act(active_gate), - act(active_state)); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); value.gateValue[frameIdx] = rValueIn; value.gateValue[frameIdx + frameSize] = rValueIg; @@ -145,11 +142,10 @@ __global__ void KeLstmBackward(Op op, LstmMetaValue value, rPrevState = value.prevStateValue[frameIdx]; } - hppl::gpu::BackwardAct act; op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, rCheckOGrad, - act(active_node), act(active_gate), act(active_state)); + active_node, active_gate, active_state); grad.gateGrad[frameIdx] = rGradIn; grad.gateGrad[frameIdx + frameSize] = rGradIg; @@ -230,9 +226,9 @@ void gpu_lstm_backward(const platform::DeviceContext& context, Op op, threads = dim3(framePerBlock, 1); grid = dim3(frameBlocks, 1); } else { - /* framePerBlock = 32 batchPerBlock = 32 */ - threads = dim3(32, 32); - grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 32 - 1) / 32); + /* framePerBlock = 32 batchPerBlock = 16 */ + threads = dim3(32, 16); + grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 16 - 1) / 16); } auto stream = @@ -248,6 +244,11 @@ void gpu_lstm_backward(const platform::DeviceContext& context, Op op, op, value, grad, frameSize, batchSize, active_node, active_gate, active_state); } + + cudaStreamSynchronize(stream); + // TODO(qingqing): Add cuda error check for each kernel. + cudaError_t err = cudaGetLastError(); + PADDLE_ENFORCE(err, cudaGetErrorString(err)); } } // namespace detail diff --git a/paddle/operators/math/detail/lstm_kernel.h b/paddle/operators/math/detail/lstm_kernel.h index 6f3ead2397d5131b4468d0ad288513cedb289594..9daaf91981a8e0252374f528f0e063111bd32675 100644 --- a/paddle/operators/math/detail/lstm_kernel.h +++ b/paddle/operators/math/detail/lstm_kernel.h @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/platform/hostdevice.h" #include @@ -30,15 +30,15 @@ class lstm { HOSTDEVICE void operator()(T &valueIn, T &valueIg, T &valueFg, T &valueOg, T &prevState, T &state, T &stateAtv, T &output, T &checkI, T &checkF, T &checkO, - typename hppl::ForwardActType::type actInput, - typename hppl::ForwardActType::type actGate, - typename hppl::ForwardActType::type actState) { - valueIn = actInput(valueIn); - valueIg = actGate(valueIg + prevState * checkI); - valueFg = actGate(valueFg + prevState * checkF); + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + valueIn = activation(valueIn, active_node); + valueIg = activation(valueIg + prevState * checkI, active_gate); + valueFg = activation(valueFg + prevState * checkF, active_gate); state = valueIn * valueIg + prevState * valueFg; - valueOg = actGate(valueOg + state * checkO); - stateAtv = actState(state); + valueOg = activation(valueOg + state * checkO, active_gate); + stateAtv = activation(state, active_state); output = valueOg * stateAtv; } #ifndef __NVCC__ @@ -52,16 +52,19 @@ class lstm { __m256 &valueOg, __m256 &prevState, __m256 &state, __m256 &stateAtv, __m256 &output, __m256 &checkI, __m256 &checkF, __m256 &checkO, - hppl::Active<__m256>::forward actInput, - hppl::Active<__m256>::forward actGate, - hppl::Active<__m256>::forward actState) { - valueIn = actInput(valueIn); - valueIg = actGate(_mm256_add_ps(valueIg, _mm256_mul_ps(prevState, checkI))); - valueFg = actGate(_mm256_add_ps(valueFg, _mm256_mul_ps(prevState, checkF))); + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + valueIn = activation(valueIn, active_node); + valueIg = activation( + _mm256_add_ps(valueIg, _mm256_mul_ps(prevState, checkI)), active_gate); + valueFg = activation( + _mm256_add_ps(valueFg, _mm256_mul_ps(prevState, checkF)), active_gate); state = _mm256_add_ps(_mm256_mul_ps(valueIn, valueIg), _mm256_mul_ps(prevState, valueFg)); - valueOg = actGate(_mm256_add_ps(valueOg, _mm256_mul_ps(state, checkO))); - stateAtv = actState(state); + valueOg = activation(_mm256_add_ps(valueOg, _mm256_mul_ps(state, checkO)), + active_gate); + stateAtv = activation(state, active_state); output = _mm256_mul_ps(valueOg, stateAtv); } #endif @@ -81,14 +84,15 @@ class lstm { T &stateGrad, T &stateAtv, T &outputGrad, T &checkI, T &checkF, T &checkO, T &checkIGrad, T &checkFGrad, T &checkOGrad, - typename hppl::BackwardActType::type actInput, - typename hppl::BackwardActType::type actGate, - typename hppl::BackwardActType::type actState) { - gradOg = actGate(outputGrad * stateAtv, valueOg); - stateGrad += actState(outputGrad * valueOg, stateAtv) + gradOg * checkO; - gradIn = actInput(stateGrad * valueIg, valueIn); - gradIg = actGate(stateGrad * valueIn, valueIg); - gradFg = actGate(stateGrad * prevState, valueFg); + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + gradOg = activation(outputGrad * stateAtv, valueOg, active_gate); + stateGrad += activation(outputGrad * valueOg, stateAtv, active_state) + + gradOg * checkO; + gradIn = activation(stateGrad * valueIg, valueIn, active_node); + gradIg = activation(stateGrad * valueIn, valueIg, active_gate); + gradFg = activation(stateGrad * prevState, valueFg, active_gate); prevStateGrad = gradIg * checkI + gradFg * checkF + stateGrad * valueFg; checkIGrad = gradIg * prevState; checkFGrad = gradFg * prevState; @@ -100,24 +104,26 @@ class lstm { #else // Only float support AVX optimization static const bool avx = std::is_same::value; - HOSTDEVICE void operator()(__m256 &valueIn, __m256 &valueIg, __m256 &valueFg, - __m256 &valueOg, __m256 &gradIn, __m256 &gradIg, - __m256 &gradFg, __m256 &gradOg, __m256 &prevState, - __m256 &prevStateGrad, __m256 &state, - __m256 &stateGrad, __m256 &stateAtv, - __m256 &outputGrad, __m256 &checkI, __m256 &checkF, - __m256 &checkO, __m256 &checkIGrad, - __m256 &checkFGrad, __m256 &checkOGrad, - hppl::Active<__m256>::backward actInput, - hppl::Active<__m256>::backward actGate, - hppl::Active<__m256>::backward actState) { - gradOg = actGate(_mm256_mul_ps(outputGrad, stateAtv), valueOg); + HOSTDEVICE void operator()( + __m256 &valueIn, __m256 &valueIg, __m256 &valueFg, __m256 &valueOg, + __m256 &gradIn, __m256 &gradIg, __m256 &gradFg, __m256 &gradOg, + __m256 &prevState, __m256 &prevStateGrad, __m256 &state, + __m256 &stateGrad, __m256 &stateAtv, __m256 &outputGrad, __m256 &checkI, + __m256 &checkF, __m256 &checkO, __m256 &checkIGrad, __m256 &checkFGrad, + __m256 &checkOGrad, activation_mode_t active_node, + activation_mode_t active_gate, activation_mode_t active_state) { + gradOg = + activation(_mm256_mul_ps(outputGrad, stateAtv), valueOg, active_gate); stateGrad = _mm256_add_ps( - actState(_mm256_mul_ps(outputGrad, valueOg), stateAtv), stateGrad); + activation(_mm256_mul_ps(outputGrad, valueOg), stateAtv, active_state), + stateGrad); stateGrad = _mm256_add_ps(_mm256_mul_ps(gradOg, checkO), stateGrad); - gradIn = actInput(_mm256_mul_ps(stateGrad, valueIg), valueIn); - gradIg = actGate(_mm256_mul_ps(stateGrad, valueIn), valueIg); - gradFg = actGate(_mm256_mul_ps(stateGrad, prevState), valueFg); + gradIn = + activation(_mm256_mul_ps(stateGrad, valueIg), valueIn, active_node); + gradIg = + activation(_mm256_mul_ps(stateGrad, valueIn), valueIg, active_gate); + gradFg = + activation(_mm256_mul_ps(stateGrad, prevState), valueFg, active_gate); prevStateGrad = _mm256_add_ps(_mm256_mul_ps(gradIg, checkI), _mm256_mul_ps(gradFg, checkF)); prevStateGrad = diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index aad1357598c629a4edfe0ad9b23f0241093a2522..2a9c09a0f16b71473e21765ab9253eb7b8bcf28c 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -211,6 +211,26 @@ void batched_gemm( } #endif +template <> +void gemv(const platform::DeviceContext& context, + const bool trans_a, const int M, + const int N, const float alpha, + const float* A, const float* B, + const float beta, float* C) { + CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; + cblas_sgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); +} + +template <> +void gemv(const platform::DeviceContext& context, + const bool trans_a, const int M, + const int N, const double alpha, + const double* A, const double* B, + const double beta, double* C) { + CBLAS_TRANSPOSE transA = (trans_a == false) ? CblasNoTrans : CblasTrans; + cblas_dgemv(CblasRowMajor, transA, M, N, alpha, A, N, B, 1, beta, C, 1); +} + template struct SetConstant; } // namespace math diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index 5583683c6e12b88ba81015aef9161913de261ef2..e6fd8bf235b8539702ca2c5b39e305cb1becf5cb 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -203,6 +203,33 @@ void batched_gemm( &beta, C, ldc, strideC, batchCount)); } +template <> +void gemv(const platform::DeviceContext& context, + const bool trans_a, const int M, + const int N, const float alpha, + const float* A, const float* B, + const float beta, float* C) { + cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; + + PADDLE_ENFORCE(platform::dynload::cublasSgemv( + reinterpret_cast(context) + .cublas_handle(), + cuTransA, N, M, &alpha, A, N, B, 1, &beta, C, 1)); +} + +template <> +void gemv(const platform::DeviceContext& context, + const bool trans_a, const int M, + const int N, const double alpha, + const double* A, const double* B, + const double beta, double* C) { + cublasOperation_t cuTransA = (trans_a == false) ? CUBLAS_OP_T : CUBLAS_OP_N; + PADDLE_ENFORCE(platform::dynload::cublasDgemv( + reinterpret_cast(context) + .cublas_handle(), + cuTransA, N, M, &alpha, A, N, B, 1, &beta, C, 1)); +} + template struct SetConstant; } // namespace math diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 9777ebfd156709a370be2cb4ba0077ac7c6735fb..3bb5aa0332c7e2a63d20b91893c03ccd468dd863 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -93,6 +93,11 @@ void batched_gemm(const platform::DeviceContext& context, const T* A, const T* B, const T beta, T* C, const int batchCount, const int strideA, const int strideB); +template +void gemv(const platform::DeviceContext& context, const bool trans_a, + const int M, const int N, const T alpha, const T* A, const T* B, + const T beta, T* C); + template struct SetConstant { void operator()(const platform::DeviceContext& context, diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 3b9f92e7ae5f34dd0fb1ba8fb0c67ff5ae1628c4..7d84ad9aadb2892db0d0ee9cab428dc5036614e9 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -89,3 +89,53 @@ TEST(math_function, zero) { EXPECT_EQ(t[2], 1); EXPECT_EQ(t[3], 1); } + +template +void GemvTest(int m, int n, bool trans) { + paddle::framework::Tensor mat_a; + paddle::framework::Tensor vec_b; + paddle::framework::Tensor vec_c; + auto* cpu_place = new paddle::platform::CPUPlace(); + int b_num = trans ? m : n; + int c_num = trans ? n : m; + + T* data_a = mat_a.mutable_data({m, n}, *cpu_place); + T* data_b = vec_b.mutable_data({b_num}, *cpu_place); + T* data_c = vec_c.mutable_data({c_num}, *cpu_place); + for (int i = 0; i < mat_a.numel(); ++i) { + data_a[i] = static_cast(i); + } + for (int i = 0; i < vec_b.numel(); ++i) { + data_b[i] = static_cast(i); + } + + paddle::platform::CPUDeviceContext context(*cpu_place); + paddle::operators::math::gemv( + context, trans, static_cast(m), static_cast(n), 1., data_a, + data_b, 0., data_c); + + if (!trans) { + for (int i = 0; i < m; ++i) { + T sum = 0.0; + for (int j = 0; j < n; ++j) { + sum += data_a[i * n + j] * data_b[j]; + } + ASSERT_FLOAT_EQ(data_c[i], sum); + } + } else { + for (int i = 0; i < n; ++i) { + T sum = 0.0; + for (int j = 0; j < m; ++j) { + sum += data_a[j * n + i] * data_b[j]; + } + ASSERT_FLOAT_EQ(data_c[i], sum); + } + } +} + +TEST(math_function, gemv) { + GemvTest(3, 13, false); + GemvTest(4, 5, false); + GemvTest(12, 7, true); + GemvTest(7, 9, true); +} diff --git a/paddle/operators/math/math_function_test.cu b/paddle/operators/math/math_function_test.cu index 8b22c71552a65044cbd02441fb35c1eafe0173dc..780d17ffc6539c5f4d67ebab5476d6f646840b41 100644 --- a/paddle/operators/math/math_function_test.cu +++ b/paddle/operators/math/math_function_test.cu @@ -177,3 +177,65 @@ TEST(math_function, gemm_trans_cublas) { EXPECT_EQ(input3_ptr[7], 99); delete gpu_place; } + +template +void GemvTest(int m, int n, bool trans) { + paddle::framework::Tensor mat_a; + paddle::framework::Tensor vec_b; + paddle::framework::Tensor vec_c; + auto* cpu_place = new paddle::platform::CPUPlace(); + + T* data_a = mat_a.mutable_data({m, n}, *cpu_place); + T* data_b = vec_b.mutable_data({trans ? m : n}, *cpu_place); + T* data_c = vec_c.mutable_data({trans ? n : m}, *cpu_place); + + auto* gpu_place = new paddle::platform::GPUPlace(0); + paddle::framework::Tensor g_mat_a; + paddle::framework::Tensor g_vec_b; + paddle::framework::Tensor g_vec_c; + T* g_data_a = g_mat_a.mutable_data(mat_a.dims(), *gpu_place); + T* g_data_b = g_vec_b.mutable_data(vec_b.dims(), *gpu_place); + T* g_data_c = g_vec_c.mutable_data(vec_c.dims(), *gpu_place); + + for (int i = 0; i < mat_a.numel(); ++i) { + data_a[i] = static_cast(i); + } + for (int i = 0; i < vec_b.numel(); ++i) { + data_b[i] = static_cast(i); + } + + paddle::platform::CUDADeviceContext context(*gpu_place); + g_mat_a.CopyFrom(mat_a, *gpu_place, context); + g_vec_b.CopyFrom(vec_b, *gpu_place, context); + + paddle::operators::math::gemv( + context, trans, static_cast(m), static_cast(n), 1., g_data_a, + g_data_b, 0., g_data_c); + + vec_c.CopyFrom(g_vec_c, paddle::platform::CPUPlace(), context); + + if (!trans) { + for (int i = 0; i < m; ++i) { + T sum = 0.0; + for (int j = 0; j < n; ++j) { + sum += data_a[i * n + j] * data_b[j]; + } + ASSERT_FLOAT_EQ(data_c[i], sum); + } + } else { + for (int i = 0; i < n; ++i) { + T sum = 0.0; + for (int j = 0; j < m; ++j) { + sum += data_a[j * n + i] * data_b[j]; + } + ASSERT_FLOAT_EQ(data_c[i], sum); + } + } +} + +TEST(math_function, gemv) { + GemvTest(3, 13, false); + GemvTest(3, 13, false); + GemvTest(3, 13, true); + GemvTest(3, 13, true); +} diff --git a/paddle/operators/math/sequence2batch.h b/paddle/operators/math/sequence2batch.h index 03cd018e46e90c9bbe689c9686377e0e998ee513..b1ba35a6d4a891e9152ac2088bc76e3969be6405 100644 --- a/paddle/operators/math/sequence2batch.h +++ b/paddle/operators/math/sequence2batch.h @@ -53,7 +53,18 @@ class LoDTensor2BatchFunctor { public: void operator()(const platform::DeviceContext& context, const framework::LoDTensor& lod_tensor, - framework::LoDTensor& batch, bool is_reverse) const { + framework::LoDTensor& batch, bool is_cal_batch_lod, + bool is_reverse = false) const { + if (!is_cal_batch_lod) { + auto lods = batch.lod(); + PADDLE_ENFORCE_EQ(lods.size(), 2UL); + PADDLE_ENFORCE_EQ(lods[1].size(), + static_cast(lod_tensor.dims()[0])); + CopyMatrixRowsFunctor to_batch; + to_batch(context, lod_tensor, lods[1].data(), batch, true); + return; + } + auto lods = lod_tensor.lod(); PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); auto lod = lods[0]; @@ -101,10 +112,10 @@ class LoDTensor2BatchFunctor { size_t* batch_starts = batch_lods[0].data(); size_t* seq2batch_idx = batch_lods[1].data(); batch_starts[0] = 0; - for (size_t n = 0; n < num_batch; n++) { + for (int n = 0; n < num_batch; n++) { auto batch_id = static_cast(batch_starts[n]); for (size_t i = 0; i < seq_info.size(); ++i) { - size_t seq_len = seq_info[i].length; + int seq_len = seq_info[i].length; int start = seq_info[i].start; if (n < seq_len) { seq2batch_idx[batch_id] = @@ -132,11 +143,8 @@ class Batch2LoDTensorFunctor { auto in_lod = batch.lod(); PADDLE_ENFORCE_EQ(in_lod.size(), 2UL, "The LoD size of input `batch` should be 2."); - auto out_lod = lod_tensor.lod()[0]; - auto num = out_lod[out_lod.size() - 1]; - PADDLE_ENFORCE_EQ(num, lod_tensor.dims()[0]); - PADDLE_ENFORCE_EQ(num, in_lod[1].size()); - PADDLE_ENFORCE_EQ(num, batch.dims()[0]); + PADDLE_ENFORCE_EQ(in_lod[1].size(), + static_cast(lod_tensor.dims()[0])); CopyMatrixRowsFunctor to_seq; size_t* index = in_lod[1].data(); to_seq(context, batch, index, lod_tensor, false); diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 5ecbee3b413617e3a5523d9a32e72bc08bd316c5..5a1a6154203d40186f1e41491194b19612931b1f 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -144,7 +144,10 @@ class MatMulOpMaker : public framework::OpProtoAndCheckerMaker { )DOC") .SetDefault(false); AddComment(R"DOC( -The MatMul operator is used to perform (batched) matrix multiplication +MatMul Operator. + + +This operator is used to perform (batched) matrix multiplication over the last two dimensions of the input tensors `X` and `Y`. If a transpose flag is specified, the last two dimensions of the @@ -166,7 +169,8 @@ The differences are: - We add `transpose_X` and `transpose_Y` flags. Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 7caa1c9d0cf4dba33a206c85bcbed1fb1cb4e010..78b4bbca84d4670aba73222f1d679604d7516b02 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -36,7 +36,11 @@ class MeanOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of mean op"); AddOutput("Out", "The output of mean op"); - AddComment(R"DOC( Mean Operator + AddComment(R"DOC( +Mean Operator. + +Out is a scalar which is the mean of all elements in X. + )DOC"); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index f7943e99acc5975d077f2319b6f678cfc693c1f3..4684c20208501a3239fd57b35428946bb52af4a0 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -52,14 +52,16 @@ class MinusOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Y", "The right tensor of minus operator."); AddOutput("Out", "The output tensor of minus operator."); - AddComment(R"DOC(Minus Operator + AddComment(R"DOC( +Minus Operator. Equation: - Out = X - Y + $Out = X - Y$ Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index 7b9e9528952d552a69ffe6a628672901c5c1a7fd..28528848af1f467bf38be53f9d05fee6ca3f93cc 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -43,27 +43,35 @@ class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "The input tensor of modified huber loss op." + "The input tensor of modified huber loss op. " "X is 2-D tensor with shape [batch_size, 1]."); AddInput("Y", - "The target labels of modified huber loss op." - "The shape of Y is same as X. Values of Y must be 0 or 1."); + "The target labels of modified huber loss op. " + "The shape of Y is the same as X. Values of Y must be 0 or 1."); AddOutput("IntermediateVal", "Variable to save intermediate result which will be reused in " "backward processing.") .AsIntermediate(); AddOutput("Out", "Classification loss for X."); AddComment(R"DOC( -Modified huber loss is used in binary classification problem. The shape of -input X and target Y are both [N, 1] and so is the shape of output loss. -Since target Y is not differentiable, cacluating gradient for Y is illegal. -The formulation of modified huber loss is: - -L(y, f(x)) = max(0, 1 - yf(x))^2 for yf(x) >= -1, - -4yf(x) otherwise. - -Make sure the values of target label Y are in {0, 1} here. The operator will +Modified Huber Loss Operator. + +This operator is used in binary classification problem. The shape of +input X and target Y are both [N, 1] and so is the shape of the output loss. +Since target Y is not differentiable, calculating gradient for Y is illegal. +The formula of modified huber loss is: + +$$ +L(y, f(x)) = +\begin{cases} +(\max(0, 1 - yf(x)))^2, \text{if} \ yf(x) >= -1 \\ + -4yf(x), \quad \text{otherwise} +\end{cases} +$$ + +Make sure the values of target label Y are in {0, 1} here. This operator will scale values of Y to {-1, +1} when computing losses and gradients. + )DOC"); } }; diff --git a/paddle/operators/momentum_op.cc b/paddle/operators/momentum_op.cc index 2d4d6f13720f0e6888edbddcb3243116506227ba..e8ce16f4cfcf83fd13e4d3a5318a4ae0c8c8449c 100644 --- a/paddle/operators/momentum_op.cc +++ b/paddle/operators/momentum_op.cc @@ -75,17 +75,23 @@ class MomentumOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("VelocityOut", "(Tensor) Output updated velocity"); AddAttr("mu", "(float) Momentum coefficient"); - AddAttr("useNesterov", "(bool) Use Nesterov Momentum") + AddAttr("useNesterov", + "(bool, default false) " + "Use Nesterov Momentum") .SetDefault(false); AddComment(R"DOC( - -Momentum Algorithm with a flag for Nestrov Moemntum (momentum). - -velocity = mu * velocity + gradient -if (use_nesterov): - param = param - gradient * learning_rate + mu * velocity * learning_rate -else: - param = param - learning_rate * velocity +Momentum Optimizer. + +This optimizer has a flag for Nestrov Momentum. +The update equations are as follows: + +$$ +velocity = mu * velocity + gradient \\ +if (use\_nesterov): \\ + param = param - gradient * learning\_rate + mu * velocity * learning\_rate \\ +else: \\ + param = param - learning\_rate * velocity. \\ +$$ )DOC"); } diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 245d3b47d3a6331a3cf20dbdbd972639d68cd496..3c39ae10dc50084cff284c307167c33c9208a3ce 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -29,9 +29,14 @@ class MulOpShapeInference : public framework::InferShapeBase { auto x_dims = ctx->GetInputDim("X"); auto y_dims = ctx->GetInputDim("Y"); + int x_num_col_dims = ctx->Attrs().Get("x_num_col_dims"); int y_num_col_dims = ctx->Attrs().Get("y_num_col_dims"); + VLOG(3) << "mul operator x.shape=" << x_dims << " y.shape=" << y_dims + << " x_num_col_dims=" << x_num_col_dims + << " y_num_col_dims=" << y_num_col_dims; + PADDLE_ENFORCE_GT( x_dims.size(), x_num_col_dims, "The input tensor X's rank of MulOp should be larger than " @@ -73,6 +78,7 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "The output of mul op"); AddAttr( "x_num_col_dims", + "(int, default 1) " R"DOC(mul_op can take tensors with more than two dimensions as input `X`, in that case, tensors will be reshaped to a matrix. The matrix's first dimension(column length) will be the product of tensor's last @@ -83,20 +89,24 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { .EqualGreaterThan(1); AddAttr( "y_num_col_dims", + "(int, default 1) " R"DOC(mul_op can take tensors with more than two dimensions as input `Y`, in that case, tensors will be reshaped to a matrix. Just like input `X`. )DOC") .SetDefault(1) .EqualGreaterThan(1); AddComment(R"DOC( -Mul operator is used to perform matrix multiplication for input X and Y. +Mul Operator. + +This operator is used to perform matrix multiplication for input X and Y. The equation is: - Out = X * Y + $$Out = X * Y$$ Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc index 4d86769026e4b3e3040bdcb3bc6dc2edea58b4b0..234fddcfd55ccc66f6378689dbc426499474b11f 100644 --- a/paddle/operators/multiplex_op.cc +++ b/paddle/operators/multiplex_op.cc @@ -66,7 +66,8 @@ class MultiplexOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "The candidate tensors of multiplex operator.") .AsDuplicable(); AddOutput("Out", "The output tensor of multiplex operator."); - AddComment(R"DOC(Multiplex operator + AddComment(R"DOC( +Multiplex Operator. Multiplex multiple tensors according to the index provided by the index tensor. @@ -77,10 +78,11 @@ the (Ids[i])-th tensor. For i-th row of the output tensor: -y[i] = x_{k}[i] +$$y[i] = x_{k}[i]$$ -where y is the output tensor. `x_{k}` is the k-th input tensor +where `y` is the output tensor, `x_{k}` is the k-th input tensor, and `k = Ids[i]`. + )DOC"); } }; diff --git a/paddle/operators/name_convention.md b/paddle/operators/name_convention.md index 5a216907950100070ba57176c382eb659effb293..b5cb176e003b4584321142ac9f1c3380b7010936 100644 --- a/paddle/operators/name_convention.md +++ b/paddle/operators/name_convention.md @@ -4,10 +4,10 @@ To make the operator document itself more clear, we recommend operator names obe ### OpProtoMaker names -When defining an operator in Paddle, a corresponding [OpProtoMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L170) (TODO: OpProtoMaker Doc)need to be defined. All the Input/Output and Attributes will write into the [OpProto](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L61) , and will be used in client language to create operator. +When defining an operator in Paddle, a corresponding [OpProtoMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L170) (TODO: OpProtoMaker Doc)need to be defined. All the Input/Output and Attributes will write into the [OpProto](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L61) , and will be used in client language to create operator. - Input/Output. - - Input/Output names follow the **CamelCase**. e.g. `X`, `Y`, `Matrix`, `LastAxisInMatrix`. Input/Output much more like Variables, we prefer to meaningful English words. + - Input/Output names follow the **CamelCase**. e.g. `X`, `Y`, `Matrix`, `LastAxisInMatrix`. Input/Output much more like Variables, we prefer to meaningful English words. - If an operator's Input/Output are tensors in math, not match to any meaningful words, input name should starts from `X`. e.g. `X`, `Y`, and output name should starts from `Out`. e.g. `Out`. This rule intends making operators which have few inputs/outputs unified. - Attribute. @@ -15,7 +15,7 @@ When defining an operator in Paddle, a corresponding [OpProtoMaker](https://gith - Comments. - Input/Output/Attr comment follow the format of **(type,default value) usage**, corresponding to which type it can be and how it will be used in the operator. e.g. Attribute in Accumulator`"gamma" `,`(float, default 1.0) Accumulation multiplier`. - - Operator comment format of` R"DOC(your comment here)DOC"`. You should explain the input/output of the operator first. If there is math calculation in this operator, you should write the equation in the comment. e.g. `Out = X + Y`. + - Operator comment format of` R"DOC(your comment here)DOC"`. You should explain the input/output of the operator first. If there is math calculation in this operator, you should write the equation in the comment. e.g. `Out = X + Y`. - Order. - Follow the order of Input/Output, then Attribute, then Comments. See the example in best practice. @@ -24,7 +24,7 @@ When defining an operator in Paddle, a corresponding [OpProtoMaker](https://gith Here we give some examples to show how these rules will be used. -- The operator has one input, one output. e.g.`relu`, inputs: `X`, outputs: `Out`. +- The operator has one input, one output. e.g.`relu`, inputs: `X`, outputs: `Out`. - The operator has two input, one output. e.g. `rowwise_add`, inputs : `X`, `Y`, outputs : `Out`. @@ -38,23 +38,27 @@ public: AccumulateOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "(Tensor) The input tensor that has to be accumulated to the output tensor. - If the output size is not the same as input size, + AddInput("X", "(Tensor) The input tensor that has to be accumulated to the output tensor. + If the output size is not the same as input size, the output tensor is first reshaped and initialized to zero, and only then, accumulation is done."); AddOutput("Out", "(Tensor) Accumulated output tensor"); AddAttr("gamma", "(float, default 1.0) Accumulation multiplier").SetDefault(1.0f); AddComment(R"DOC( -Accumulate operator accumulates the input tensor to the output tensor. If the +Accumulate Operator. + +This operator accumulates the input tensor to the output tensor. If the output tensor already has the right size, we add to it; otherwise, we first initialize the output tensor to all zeros, and then do accumulation. Any further calls to the operator, given that no one else fiddles with the output in the interim, will do simple accumulations. -Accumulation is done as shown: + +Accumulation is done as follows: Out = 1*X + gamma*Out where X is the input tensor, Out is the output tensor and gamma is the multiplier argument. + )DOC"); } }; diff --git a/paddle/operators/nccl_op.cc b/paddle/operators/nccl_op.cc index d39cb2fcf9cc205edf86f8ab1d5e04b5672e00f6..66fcc09bc877867e66a37adc73230d8dabf4cbed 100644 --- a/paddle/operators/nccl_op.cc +++ b/paddle/operators/nccl_op.cc @@ -48,12 +48,17 @@ class NCCLInitOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddOutput("Communicator", "Create Communicator for communicating between gpus"); - AddAttr>("gpus", "gpu id lists"); - AddAttr("data_type", "output data type") + AddAttr>("gpus", "(vector) GPU id lists"); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") .SetDefault(framework::DataType::FP32); AddComment(R"DOC( - create communicator. - )DOC"); +NCCLInit Operator. + +Create communicator. + +)DOC"); } }; @@ -143,11 +148,15 @@ class NCCLAllReduceOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of AllReduce op"); AddAttr("reduction", + "(string, default 'ncclSum') " "{'ncclMin', 'ncclMax', 'ncclProd', 'ncclSum'}.") .SetDefault("ncclSum"); AddComment(R"DOC( - AllReduce the input tensors. - )DOC"); +NCCLAllReduce Operator. + +AllReduce the input tensors. + +)DOC"); } }; @@ -161,14 +170,20 @@ class NCCLReduceOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of Reduce op"); AddAttr("reduction", + "(string, default 'ncclSum') " "{'ncclMin', 'ncclMax', 'ncclProd', 'ncclSum'}.") .SetDefault("ncclSum"); AddAttr("root", - "root gpu of the parameter. if not " - "set(platform::kInvalidGPUId). hashed by name.") + "(int, default kInvalidGPUId) " + "Root gpu of the parameter. If not, " + "set(platform::kInvalidGPUId). Hashed by name.") .SetDefault(platform::kInvalidGPUId); AddComment(R"DOC( - Reduce the tensors)DOC"); +NCCLReduce Operator. + +Reduce the tensors. + +)DOC"); } }; @@ -182,12 +197,16 @@ class NCCLBcastOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of Bcast"); AddAttr("root", - "root gpu of the parameter. if not " - "set(platform::kInvalidGPUId). hashed by name.") + "(int, default kInvalidGPUId) " + "Root gpu of the parameter. If not, " + "set(platform::kInvalidGPUId). Hashed by name.") .SetDefault(platform::kInvalidGPUId); AddComment(R"DOC( - Bcast the tensors. - )DOC"); +NCCLBcast Operator. + +Bcast the tensors. + +)DOC"); } }; diff --git a/paddle/operators/nccl_op_test.cu b/paddle/operators/nccl_op_test.cu index 80c50a28a9e5d560fc693c518b9e62091ddc5724..e5927d56ae7cfbd09e941c993041af46ecd8d70d 100644 --- a/paddle/operators/nccl_op_test.cu +++ b/paddle/operators/nccl_op_test.cu @@ -185,7 +185,7 @@ TEST_F(NCCLTester, ncclAllReduceOp) { recv_tensor.numel() * sizeof(float), static_cast(dev_ctxs[i])->stream()); - for (size_t j = 0; j < f::product(kDims); ++j) { + for (int64_t j = 0; j < f::product(kDims); ++j) { ASSERT_NEAR(ct[j], result, 1e-5); } } @@ -234,7 +234,7 @@ TEST_F(NCCLTester, ncclReduceOp) { recv_tensor.numel() * sizeof(float), static_cast(dev_ctxs[kRoot])->stream()); - for (int j = 0; j < f::product(kDims); ++j) { + for (int64_t j = 0; j < f::product(kDims); ++j) { ASSERT_NEAR(ct[j], result, 1e-5); } } @@ -282,7 +282,7 @@ TEST_F(NCCLTester, ncclBcastOp) { recv_tensor.numel() * sizeof(float), static_cast(dev_ctxs[idx])->stream()); - for (size_t j = 0; j < f::product(kDims); ++j) { + for (int64_t j = 0; j < f::product(kDims); ++j) { ASSERT_NEAR(ct[j], result, 1e-5); } } diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 73a0b8baff530840ddd0d4c65cd4c060ab18e401..adb75df6ef10c59fc6f3db4d36e1ffb1ae0b4b1e 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -54,41 +54,44 @@ class PadOpMaker : public framework::OpProtoAndCheckerMaker { "The input of pad op. " "The input should be a k-D tensor(k > 0 and k < 7)"); AddOutput("Out", - "The output of pad op." + "The output of pad op. " "A tensor with the same shape as X."); + AddAttr>( + "paddings", + "(vector) " + "A list to describe the padding rules for each dimension. " + "For 2-D image tensor, paddings=[0, 1, 2, 3] means " + "padding 0 row to top, 1 row to bottom, 2 columns to left " + "and 3 columns to right. Size of paddings should be equal to " + "2 * dimension size of the input tensor."); + AddAttr("pad_value", + "(float, default 0.0) " + "The value to fill the padded areas.") + .SetDefault(0.0f); AddComment(R"DOC( -Pad input into output, as specified by paddings and pad_value. The input should be a k-D tensor(k > 0 and k < 7). As an example: +Pad Operator. + +Pad input into output, as specified by paddings and pad_value. +The input should be a k-D tensor(k > 0 and k < 7). As an example: Given: X = [[1, 2], - [3, 4]] - -and + [3, 4]], -paddings = [0, 1, 1, 2] +paddings = [0, 1, 1, 2], and -pad_value = 0 +pad_value = 0, -then we get +we have: Out = [[0, 1, 2, 0, 0] [0, 3, 4, 0, 0] [0, 0, 0, 0, 0]] + )DOC"); - AddAttr>( - "paddings", - "A list to describes padding rules for each dimension." - " For 2-D image tensor, paddings=[0, 1, 2, 3] means" - " padding 0 row to top, 1 row to bottom, 2 columns to left" - " and 3 columns to right.Size of paddings should be equal to" - " 2 * dimension size of input tensor."); - AddAttr("pad_value", - "(float) default to 0; " - "The value to fill padded areas.") - .SetDefault(0.0f); } }; diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 4d75c11bc8130343e95f75e687529303179caa93..f58aab733866973f477ca79e5e53ba58adbf3dc7 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -73,125 +73,138 @@ Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto, AddInput( "X", "(Tensor) The input tensor of pooling operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of feature."); + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of channels, H is the height of the feature, " + "and W is the width of the feature."); AddOutput("Out", - "(Tensor) The output tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is " - "the number of channels, H and W is the height and " - "width of feature."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCHW, " + "where N is batch size, C is the number of channels, " + "H is the height of the feature, " + "and W is the width of the feature."); AddAttr("poolingType", "(string), pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); AddAttr>("ksize", - "(vector ), the pooling window size(height, width) " - "of pooling operator." + "(vector) The pooling window " + "size(height, width) of the pooling operator. " "If globalPooling = true, ksize and paddings will " "be ignored."); // TODO(Chengduo): Add checker. // (Currently, // TypedAttrChecker don't support vector type.) AddAttr("globalPooling", - "(bool default: false), whether to use the global pooling." + "(bool, default false) Whether to use the global pooling. " "If globalPooling = true, ksize and paddings will be ignored.") .SetDefault(false); - AddAttr>( - "strides", - "(vector, default:{1, 1}), strides(height, width) of pooling operator.") + AddAttr>("strides", + "(vector, default {1, 1}), strides(height, " + "width) of pooling operator.") .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0}), paddings(height, width) of pooling operator." + "(vector, defalut {0,0}), paddings(height, width) of pooling " + "operator." "If globalPooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +Pool2d Operator. + The pooling2d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. -Input(X) and output(Out) are in NCHW format. Where N is batch size, C is the -number of channels, H and W is the height and width of feature. +Input(X) and output(Out) are in NCHW format, where N is batch size, C is the +number of channels, H is the height of the feature, and W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively. The input(X) size and output(Out) size may be different. Example: Input: - X shape: (N, C, H_in, W_in) + X shape: $(N, C, H_{in}, W_{in})$ Output: - Out shape: (N, C, H_out, W_out) - where - H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + Out shape: $(N, C, H_{out}, W_{out})$ + where + $$ + H_{out} = (H_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + W_{out} = (W_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + $$ + )DOC"); } Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor) The input tensor of pooling operator. " - "The format of input tensor is NCDHW. Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and width of " - "feature."); + AddInput("X", + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCDHW, where N is batch size, C is " + "the number of channels, and D, H and W is the depth, height and " + "width of " + "the feature, respectively."); AddOutput("Out", "(Tensor) The output tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and " - "width of feature."); + "The format of output tensor is also NCDHW, " + "where N is batch size, C is " + "the number of channels, and D, H and W is the depth, height and " + "width of the feature, respectively."); AddAttr("poolingType", - "(string), pooling type, can be \"max\" for max-pooling " + "(string) Pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); - AddAttr>("ksize", - "(vector ), the pooling window size(depth, height, " - "width) of pooling " - "operator." - "If globalPooling = true, ksize and paddings wille " - "be ignored."); // TODO(Chengduo): Add checker. - // (Currently, + AddAttr>( + "ksize", + "(vector) The pooling window size(depth, height, " + "width) of pooling operator. " + "If globalPooling = true, ksize and paddings will " + "be ignored."); // TODO(Chengduo): Add checker. + // (Currently, // TypedAttrChecker don't support vector type.) AddAttr("globalPooling", - "(bool default: false), whether to use the global pooling." + "(bool, default false) Whether to use the global pooling. " "If globalPooling = true, ksize and paddings wille be ignored.") .SetDefault(false); - AddAttr>("strides", - "(vector, default:{1,1,1}), strides(depth, height, " - "width) of pooling operator.") + AddAttr>( + "strides", + "(vector, default {1,1,1}) Strides(depth, height, " + "width) of the pooling operator.") .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0,0}), paddings(depth, height, " - "width) of pooling operator." - "If globalPooling = true, ksize and paddings wille be ignored.") + "(vector, defalut {0,0,0}), paddings(depth, height, " + "width) of pooling operator. " + "If globalPooling = true, ksize and paddings will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +Pool3d Operator. + The pooling3d operation calculates the output based on -the input, poolingType and ksize, strides, paddings parameters. -Input(X) and output(Out) are in NCDHW format. Where N is batch -size, C is the number of channels, D, H and W is the depth, height and -width of feature. Parameters(ksize, strides, paddings) are three elements. -These three elements represent depth, height and width, respectively. -The input(X) size and output(Out) size may be different. +the input, poolingType, ksize, strides, and paddings parameters. +Input(X) and output(Out) are in NCDHW format, where N is batch +size, C is the number of channels, and D, H and W are the depth, height and +width of the feature, respectively. Parameters(ksize, strides, paddings) +are three elements. These three elements represent depth, height and +width, respectively. The input(X) size and output(Out) size may be different. Example: Input: - X shape: (N, C, D_in, H_in, W_in) + X shape: $(N, C, D_{in}, H_{in}, W_{in})$ Output: - Out shape: (N, C, D_out, H_out, W_out) + Out shape: $(N, C, D_{out}, H_{out}, W_{out})$ where - D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1; + $$ + D_{out} = (D_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + H_{out} = (H_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 \\ + W_{out} = (W_{in} - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + $$ + )DOC"); } } // namespace operators diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc index 95e896e7cc33b1aebe78d1af8746a25318048041..a31b3fcb7083ba5bfbcb6666c4641833ae1e69eb 100644 --- a/paddle/operators/pool_with_index_op.cc +++ b/paddle/operators/pool_with_index_op.cc @@ -89,64 +89,73 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", - "(Tensor), the input tensor of pooling operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of image."); + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of channels, H is the height of the image, " + "and W is the width of the image."); AddOutput("Out", - "(Tensor), the output tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is " - "the number of channels, H and W is the height and " - "width of image."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCHW, " + "where N is batch size, C is " + "the number of channels, H is the height of the image " + "and W is the width of the image."); AddOutput("Mask", - "(Tensor), the Mask tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is the number of channels, H and W " - "is the height and width of image." - "The value in it is the index in current feature map"); + "(Tensor) The Mask tensor of pooling operator." + "The format of output tensor is also NCHW, " + "where N is batch size, C is the number of channels, " + "H is the height of the image, " + "and W is the width of the image. " + "It represents the index in the current feature map."); AddAttr>("ksize", - "(vector ), the pooling window size(height, " - "width) of pooling operator." + "(vector) The pooling window size(height, " + "width) of pooling operator. " "If globalPooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( "globalPooling", - "(bool default: false), whether to use the global pooling." + "(bool, default false) Whether to use the global pooling. " "If globalPooling = true, ksize and paddings will be ignored.") .SetDefault(false); - AddAttr>( - "strides", - "(vector, default:{1, 1}), strides(height, width) of pooling operator.") + AddAttr>("strides", + "(vector, default {1, 1}), strides(height, " + "width) of pooling operator.") .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0, 0}), paddings(height, width) of pooling operator." + "(vector, defalut {0, 0}), paddings(height, width) of pooling " + "operator. " "If globalPooling = true, paddings and will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +MaxPool2d Operator. + The maxPooling2d with index operation calculates the output and the mask -based on the input and ksize, strides, paddings parameters. Input(X) and -output(Out, Mask) are in NCHW format. Where N is batch size, C is the -number of channels, H and W is the height and width of feature. +based on the input, ksize, strides, and paddings parameters. Input(X) and +output(Out, Mask) are in NCHW format, where N is batch size, C is the +number of channels, H is the height of the feature, +and W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively. The input(X) size and output(Out, Mask) size may be different. Example: Input: - X shape: (N, C, H_in, W_in) + X shape: $(N, C, H_{in}, W_{in})$ Output: - Out shape: (N, C, H_out, W_out) - Mask shape: (N, C, H_out, W_out) + Out shape: $(N, C, H_{out}, W_{out})$ + Mask shape: $(N, C, H_{out}, W_{out})$ where - H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + $$ + H_{out} = (H_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + W_{out} = (W_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + $$ + )DOC"); } }; @@ -156,70 +165,76 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { MaxPool3dWithIndexOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor), the input tensor of pooling operator. " - "The format of input tensor is NCDHW. Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and width of " - "image."); + AddInput("X", + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCDHW, where N is batch size, C is " + "the number of channels, and D, H and W are the depth, height and " + "width of " + "the image, respectively"); AddOutput("Out", - "(Tensor), the output tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and " - "width of image."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCDHW, " + "where N is the batch size, C is the number of channels, " + "and D, H and W are the depth, height and " + "width of the image, respectively."); AddOutput("Mask", - "(Tensor), the Mask tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is the number of channels, D, H and W " - "is the depth, height and width of image." - "The value in it is the index in current feature map"); + "(Tensor) The Mask tensor of pooling operator. " + "The format of output tensor is also NCDHW, " + "where N is the batch size, C is the number of channels, and " + "D, H and W are the depth, height and width " + "of the image, respectively. " + "It represents the index in the current feature map."); AddAttr>("ksize", - "(vector), the pooling window size(depth, " - "height, width) of pooling " - "operator." + "(vector) The pooling window size(depth, " + "height, width) of pooling operator. " "If globalPooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( "globalPooling", - "(bool default: false), whether to use the global pooling." + "(bool, default false) Whether to use the global pooling. " "If globalPooling = true, ksize and paddings will be ignored.") .SetDefault(false); AddAttr>("strides", - "(vector, default:{1,1,1}), strides(depth, " + "(vector, default {1,1,1}), strides(depth, " "height, width) of pooling operator.") .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0,0}), paddings(depth, " - "height, width) of pooling operator." + "(vector, defalut {0,0,0}), paddings(depth, " + "height, width) of pooling operator. " "If globalPooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +MaxPool3d Operator. + The maxpooling3d with index operation calculates the output and the mask based on the input and ksize, strides, paddings parameters. -Input(X) and output(Out, Mask) are in NCDHW format. Where N is batch -size, C is the number of channels, D, H and W is the depth, height and -width of feature. Parameters(ksize, strides, paddings) are three elements. +Input(X) and output(Out, Mask) are in NCDHW format, where N is batch +size, C is the number of channels, and D, H and W are the depth, height and +width of the feature, respectively. +Parameters(ksize, strides, paddings) are three elements. These three elements represent depth, height and width, respectively. The input(X) size and output(Out, Mask) size may be different. Example: Input: - X shape: (N, C, D_in, H_in, W_in) + X shape: $(N, C, D_{in}, H_{in}, W_{in})$ Output: - Out shape: (N, C, D_out, H_out, W_out) - Mask shape: (N, C, D_out, H_out, W_out) + Out shape: $(N, C, D_{out}, H_{out}, W_{out})$ + Mask shape: $(N, C, D_{out}, H_{out}, W_{out})$ where - D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1; + $$ + D_{out} = (D_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + H_{out} = (H_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 \\ + W_{out} = (W_{in} - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + $$ + )DOC"); } }; diff --git a/paddle/operators/precision_recall_op.cc b/paddle/operators/precision_recall_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..641f7135ded159b1c7330e87c4b1d983e959b466 --- /dev/null +++ b/paddle/operators/precision_recall_op.cc @@ -0,0 +1,181 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/precision_recall_op.h" + +namespace paddle { +namespace operators { + +class PrecisionRecallOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("MaxProbs"), + "Input(MaxProbs) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Indices"), + "Input(Indices) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Labels"), + "Input(Labels) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchMetrics"), + "Output(BatchMetrics) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("AccumMetrics"), + "Output(AccumMetrics) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("AccumStatesInfo"), + "Output(AccumStatesInfo) should not be null."); + + int64_t cls_num = + static_cast(ctx->Attrs().Get("class_number")); + auto max_probs_dims = ctx->GetInputDim("MaxProbs"); + auto labels_dims = ctx->GetInputDim("Labels"); + + PADDLE_ENFORCE_EQ(max_probs_dims[1], 1, + "Each instance contains one max probability, so the " + "shape of Input(MaxProbs) should be [batch_size, 1]."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("Indices"), max_probs_dims, + "The shape of Input(Indices) should be [batch_size, 1]."); + PADDLE_ENFORCE_EQ(max_probs_dims[0], labels_dims[0], + "The 1st dimension of Input(MaxProbs) and " + "Input(Labels) both are batch_size and the shape should " + "be the same."); + PADDLE_ENFORCE_EQ(labels_dims[1], 1, + "The 2nd dimension of Input(Labels) contains instance " + "label and the shape should be equal to 1."); + if (ctx->HasInput("Weights")) { + auto weights_dims = ctx->GetInputDim("Weights"); + PADDLE_ENFORCE_EQ(weights_dims, + framework::make_ddim({max_probs_dims[0], 1}), + "The shape of Input(Weights) should be " + "[batch_size, 1]."); + } + if (ctx->HasInput("StatesInfo")) { + auto states_dims = ctx->GetInputDim("StatesInfo"); + PADDLE_ENFORCE_EQ(states_dims, framework::make_ddim({cls_num, 4}), + "The shape of Input(StatesInfo) should be " + "[class_number, 4]."); + } + + // Layouts of BatchMetrics and AccumMetrics both are: + // [ + // macro average precision, macro average recall, macro average F1 score, + // micro average precision, micro average recall, micro average F1 score + // ] + ctx->SetOutputDim("BatchMetrics", {6}); + ctx->SetOutputDim("AccumMetrics", {6}); + // Shape of AccumStatesInfo is [class_number, 4] + // The layout of each row is: + // [ TP, FP, TN, FN ] + ctx->SetOutputDim("AccumStatesInfo", {cls_num, 4}); + } + + protected: + framework::DataType IndicateDataType( + const framework::ExecutionContext &ctx) const override { + return framework::ToDataType(ctx.Input("MaxProbs")->type()); + } +}; + +class PrecisionRecallOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PrecisionRecallOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("MaxProbs", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each row contains the max probability " + "of an instance which computed by the previous top_k (k=1) " + "operator."); + AddInput("Indices", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each row contains the corresponding " + "index which computed by the previous top_k (k=1) operator."); + AddInput("Labels", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each element is a label and the " + "value should be in [0, class_number - 1]."); + AddInput("Weights", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. This input is optional. If provided, " + "weight of instance would be considered when computing metrics.") + .AsDispensable(); + AddInput("StatesInfo", + "(Tensor, default Tensor) A 2-D tensor with shape D x 4, " + "where D is the number of classes. This input is optional. If " + "provided, current state will be accumulated to this state and " + "the accumulation state will be the output state.") + .AsDispensable(); + AddOutput("BatchMetrics", + "(Tensor, default Tensor) A 1-D tensor with shape {6}. " + "This output tensor contains metrics for current batch data. " + "The layout is [macro average precision, macro average recall, " + "macro f1 score, micro average precision, micro average recall, " + "micro f1 score]."); + AddOutput("AccumMetrics", + "(Tensor, default Tensor) A 1-D tensor with shape {6}. " + "This output tensor contains metrics for accumulated data. " + "The layout is [macro average precision, macro average recall, " + "macro f1 score, micro average precision, micro average recall, " + "micro f1 score]."); + AddOutput("AccumStatesInfo", + "(Tensor, default Tensor) A 2-D tensor with shape D x 4, " + "where D is equal to class number. This output tensor contains " + "accumulated state variables used to compute metrics. The layout " + "for each class is [true positives, false positives, " + "true negatives, false negatives]."); + AddAttr("class_number", "(int) Number of classes to be evaluated."); + AddComment(R"DOC( +Precision Recall Operator. + +When given Input(Indices) and Input(Labels), this operator can be used +to compute various metrics including: +1. macro average precision +2. macro average recall +3. macro f1 score +4. micro average precision +5. micro average recall +6. micro f1 score + +To compute the above metrics, we need to do statistics for true positives, +false positives and false negatives. Here the count of true negatives is not +necessary, but counting it may provide potential usage and the cost is +trivial, so the operator also provides the count of true negatives. + +We define state as a 2-D tensor with shape [class_number, 4]. Each row of a +state contains statistic variables for corresponding class. Layout of each row +is: TP(true positives), FP(false positives), TN(true negatives), +FN(false negatives). If Input(Weights) is provided, TP, FP, TN, FN will be +calculated by given weight instead of the instance count. + +This operator also supports metrics computing for cross-batch situation. To +achieve this, Input(StatesInfo) should be provided. State of current batch +data will be accumulated to Input(StatesInfo) and Output(AccumStatesInfo) +is the accumulation state. + +Output(BatchMetrics) is metrics of current batch data while +Output(AccumStatesInfo) is metrics of accumulation data. + +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(precision_recall, ops::PrecisionRecallOp, + ops::PrecisionRecallOpMaker); +REGISTER_OP_CPU_KERNEL( + precision_recall, + ops::PrecisionRecallKernel, + ops::PrecisionRecallKernel); diff --git a/paddle/operators/precision_recall_op.h b/paddle/operators/precision_recall_op.h new file mode 100644 index 0000000000000000000000000000000000000000..4a871ce6741469cf9af409ec90215f721d52f36c --- /dev/null +++ b/paddle/operators/precision_recall_op.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +enum StateVariable { TP = 0, FP, TN, FN }; + +template +class PrecisionRecallKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* in0 = ctx.Input("Indices"); + auto* in1 = ctx.Input("Labels"); + auto* in2 = ctx.Input("Weights"); + auto* in3 = ctx.Input("StatesInfo"); + auto* out0 = ctx.Output("BatchMetrics"); + auto* out1 = ctx.Output("AccumMetrics"); + auto* out2 = ctx.Output("AccumStatesInfo"); + + const int* ids_data = in0->data(); + const int* labels_data = in1->data(); + size_t cls_num = static_cast(ctx.Attr("class_number")); + const T* weights_data = in2 ? in2->data() : nullptr; + const T* states_data = in3 ? in3->data() : nullptr; + double* batch_metrics_data = out0->mutable_data(ctx.GetPlace()); + double* accum_metrics_data = out1->mutable_data(ctx.GetPlace()); + out2->mutable_data(ctx.GetPlace()); + auto accum_states = EigenMatrix::From(*out2); + accum_states.setZero(); + T* accum_states_data = out2->data(); + + size_t sample_num = in0->dims()[0]; + size_t state_var_num = 4; // TP FP TN FN + + // get states info for current batch + for (size_t i = 0; i < sample_num; ++i) { + size_t idx = ids_data[i]; + size_t label = labels_data[i]; + + PADDLE_ENFORCE(idx >= 0 && idx < cls_num, + "Class index of each instance should be in " + "[0, class_number)."); + PADDLE_ENFORCE(label >= 0 && label < cls_num, + "Label of each instance should be in [0, class_number)."); + + T w = weights_data ? weights_data[i] : 1.0; + if (idx == label) { + accum_states_data[idx * state_var_num + TP] += w; + for (size_t j = 0; j < cls_num; ++j) { + accum_states_data[j * state_var_num + TN] += w; + } + accum_states_data[idx * state_var_num + TN] -= w; + } else { + accum_states_data[label * state_var_num + FN] += w; + accum_states_data[idx * state_var_num + FP] += w; + for (size_t j = 0; j < cls_num; ++j) { + accum_states_data[j * state_var_num + TN] += w; + } + accum_states_data[idx * state_var_num + TN] -= w; + accum_states_data[label * state_var_num + TN] -= w; + } + } + + ComputeMetrics(accum_states_data, batch_metrics_data, state_var_num, + cls_num); + + if (states_data) { + for (size_t i = 0; i < cls_num; ++i) { + for (size_t j = 0; j < state_var_num; ++j) { + size_t idx = i * state_var_num + j; + accum_states_data[idx] += states_data[idx]; + } + } + } + + ComputeMetrics(accum_states_data, accum_metrics_data, state_var_num, + cls_num); + } + + // expose to be reused + static inline T CalcPrecision(T tp_count, T fp_count) { + if (tp_count > 0.0 || fp_count > 0.0) { + return tp_count / (tp_count + fp_count); + } + return 1.0; + } + + static inline T CalcRecall(T tp_count, T fn_count) { + if (tp_count > 0.0 || fn_count > 0.0) { + return tp_count / (tp_count + fn_count); + } + return 1.0; + } + + static inline T CalcF1Score(T precision, T recall) { + if (precision > 0.0 || recall > 0.0) { + return 2 * precision * recall / (precision + recall); + } + return 0.0; + } + + protected: + void ComputeMetrics(const T* states_data, double* metrics_data, + size_t state_var_num, size_t cls_num) const { + T total_tp_count = 0; + T total_fp_count = 0; + T total_fn_count = 0; + T macro_avg_precision = 0.0; + T macro_avg_recall = 0.0; + + for (size_t i = 0; i < cls_num; ++i) { + T tp_count = states_data[i * state_var_num + TP]; + T fp_count = states_data[i * state_var_num + FP]; + T fn_count = states_data[i * state_var_num + FN]; + total_tp_count += tp_count; + total_fp_count += fp_count; + total_fn_count += fn_count; + macro_avg_precision += CalcPrecision(tp_count, fp_count); + macro_avg_recall += CalcRecall(tp_count, fn_count); + } + macro_avg_precision /= cls_num; + macro_avg_recall /= cls_num; + T macro_f1_score = CalcF1Score(macro_avg_precision, macro_avg_recall); + + T micro_avg_precision = CalcPrecision(total_tp_count, total_fp_count); + T micro_avg_recall = CalcRecall(total_tp_count, total_fn_count); + T micro_f1_score = CalcF1Score(micro_avg_precision, micro_avg_recall); + + // fill metrics data + metrics_data[0] = macro_avg_precision; + metrics_data[1] = macro_avg_recall; + metrics_data[2] = macro_f1_score; + metrics_data[3] = micro_avg_precision; + metrics_data[4] = micro_avg_recall; + metrics_data[5] = micro_f1_score; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc index eef2e34eaacf59b9adacb343e9a0091ebabeaea3..055c471b4561e5fd3c7a65c6f81d66cdce1a5578 100644 --- a/paddle/operators/prelu_op.cc +++ b/paddle/operators/prelu_op.cc @@ -41,17 +41,24 @@ class PReluOpMaker : public framework::OpProtoAndCheckerMaker { PReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input tensor of prelu operator."); - AddInput("Alpha", "The alpha weight of PRelu operator."); - AddOutput("Out", "The output tensor of PRelu operator."); - AddComment(R"DOC(PRelu operator + AddInput("Alpha", "The alpha weight of prelu operator."); + AddOutput("Out", "The output tensor of prelu operator."); + AddComment(R"DOC( +PRelu Operator. The equation is: - f(x) = alpha * x , for x < 0 - f(x) = x , for x >= 0 +$$ +f(x) = +\begin{cases} +\alpha * x, \quad \text{if} \ x < 0 \\ +x, \qquad \text{if} \ x >= 0 +\end{cases} +$$ The input `X` can carry the LoD (Level of Details) information, -or not. And the output shares the LoD with input `X`. +or not. And the output shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/proximal_adagrad_op.cc b/paddle/operators/proximal_adagrad_op.cc index 39fbf800031cd559a49654667e5a6f634384523d..36e460103ab46bf6f1408840a0699793e2be134d 100644 --- a/paddle/operators/proximal_adagrad_op.cc +++ b/paddle/operators/proximal_adagrad_op.cc @@ -83,22 +83,26 @@ class ProximalAdagradOpMaker : public framework::OpProtoAndCheckerMaker { "L1 regularization strength.") .SetDefault(0.0f); AddAttr("l2", - "(float, default 0.0)" + "(float, default 0.0) " "L2 regularization strength.") .SetDefault(0.0f); AddComment(R"DOC( +Proximal Adagrad Optimizer. -Optimizer that implements the proximal adagrad algorithm. +Optimizer that implements the proximal adagrad algorithm: -moment = moment + grad * grad -prox_param = param - learning_rate * grad * (1 / sqrt(moment)) -param = sign(prox_param) / (1 + learning_rate * l2) * - max { |prox_param| - learning_rate * l1 , 0 } +$$ +moment = moment + grad * grad \\ +prox\_param = param - learning\_rate * grad * (1 / \sqrt{moment}) \\ +param = sign(prox\_param) / (1 + learning\_rate * l2) * + \max(|prox\_param| - learning\_rate * l1 , 0) +$$ The paper that proposed Proximal GD: (http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf) Here, we use the adagrad learning rate as specified here: (http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + )DOC"); } }; diff --git a/paddle/operators/proximal_gd_op.cc b/paddle/operators/proximal_gd_op.cc index e4b014b9f5866ec0791cba9b3998b1734066eeeb..5693d0ec9ebf4c470dfa5141b6eeee431f24f2ea 100644 --- a/paddle/operators/proximal_gd_op.cc +++ b/paddle/operators/proximal_gd_op.cc @@ -67,19 +67,23 @@ class ProximalGDOpMaker : public framework::OpProtoAndCheckerMaker { "L1 regularization strength.") .SetDefault(0.0f); AddAttr("l2", - "(float, default 0.0)" + "(float, default 0.0) " "L2 regularization strength.") .SetDefault(0.0f); AddComment(R"DOC( +ProximalGD Operator. -Optimizer that implements the proximal gradient descent algorithm. +Optimizer that implements the proximal gradient descent algorithm: -prox_param = param - learning_rate * grad -param = sign(prox_param) / (1 + learning_rate * l2) * - max { |prox_param| - learning_rate * l1 , 0 } +$$ +prox\_param = param - learning\_rate * grad \\ +param = sign(prox\_param) / (1 + learning\_rate * l2) * + \max(|prox\_param| - learning\_rate * l1, 0) +$$ The paper that proposed Proximal Gradient Descent: (http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf) + )DOC"); } }; diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc index 17ef2b1d01bd37abf2ece97ed0a307c2f1bf7e6f..061e82412ea5f4f17fd26a7094e68b97138cc09c 100644 --- a/paddle/operators/rank_loss_op.cc +++ b/paddle/operators/rank_loss_op.cc @@ -26,9 +26,9 @@ class RankLossOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext *ctx) const override { // input check - PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null"); - PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null"); - PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null"); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null."); auto label_dims = ctx->GetInputDim("Label"); auto left_dims = ctx->GetInputDim("Left"); @@ -50,32 +50,32 @@ class RankLossOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Label", "The label indicating A ranked higher than B or not, row vector."); AddInput("Left", "The output of RankNet for doc A, vector."); - AddInput("Right", "The output of RankNet for doc B, vetor"); + AddInput("Right", "The output of RankNet for doc B, vetor."); AddOutput("Out", "The output loss of RankLoss operator, vector."); - AddComment(R"DOC(RankLoss operator + AddComment(R"DOC( +RankLoss Operator. -Rank loss operator for RankNet[1]. RankNet is a pairwise ranking model with +RankLoss operator for RankNet +(http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf). +RankNet is a pairwise ranking model with one training sample consisting of a pair of doc A and B, and the label P indicating that A is ranked higher than B or not: P = {0, 1} or {0, 0.5, 1}, where 0.5 means no information about the rank of the input pair. -The RankLoss operator contains three inputs: Left (o_i), Right (o_j) and Label -(P_{i,j}), which represent the output of RankNet for two docs and the label -respectively, and yields the rank loss C_{i,j} by following the expression +The RankLoss operator takes three inputs: Left (o_i), Right (o_j) and Label +(P_{i,j}), which represent the output of RankNet for the two docs and the label, +respectively, and yields the rank loss C_{i,j} using the following equation: -\f[ +\f$$ C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + log(1 + e^{o_{i,j}}) \\ o_{i,j} = o_i - o_j \\ \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} -\f] +\f$$ The operator can take inputs of one sample or in batch. -[1]. Chris Burges, Tal Shaked, Erin Renshaw, et al. Learning to - Rank using Gradient Descent. - http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf )DOC"); } }; diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 40303e3adf4db7e8336ed72667fe69afa56c3f69..b0e87b7059eab3772c179fe31cdb09477b589ed1 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -12,181 +12,620 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/recurrent_op.h" - -#include -#include - +#include +#include "paddle/framework/executor.h" #include "paddle/framework/op_registry.h" -#include "paddle/operators/net_op.h" namespace paddle { namespace operators { +constexpr char kInputs[] = "inputs"; +constexpr char kInitialStates[] = "initial_states"; +constexpr char kParameters[] = "parameters"; +constexpr char kOutputs[] = "outputs"; +constexpr char kStepScopes[] = "step_scopes"; +constexpr char kExStates[] = "ex_states"; +constexpr char kStates[] = "states"; +constexpr char kStepBlock[] = "step_block"; +constexpr char kReverse[] = "reverse"; +constexpr char kIsTrain[] = "is_train"; +#define GRAD_SUFFIX "@GRAD" +constexpr char kInputGrads[] = "inputs" GRAD_SUFFIX; +constexpr char kOutputGrads[] = "outputs" GRAD_SUFFIX; +constexpr char kParamGrads[] = "parameters" GRAD_SUFFIX; +constexpr char kInitStateGrads[] = "initial_states" GRAD_SUFFIX; -using Scope = framework::Scope; -using Variable = framework::Variable; -using Tensor = framework::Tensor; -using LoDTensor = framework::LoDTensor; - -void RecurrentAlgorithm::Run(const Scope& scope, - const platform::DeviceContext& dev_ctx) const { - auto* input0 = scope.FindVar(arg_->inlinks[0]); - PADDLE_ENFORCE_NOT_NULL(input0); - size_t seq_len = input0->GetMutable()->dims()[0]; - PADDLE_ENFORCE_GT(seq_len, 0); - - CreateScopes(scope, seq_len); - auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); - InitMemories(step_scopes[0]); - - for (size_t step_id = 0; step_id < seq_len; step_id++) { - if (step_id > 0) { - rnn::LinkMemories(step_scopes, arg_->states, step_id, -1); +using StepScopeVar = std::vector; + +// StepScopes manages scopes inside RNN. +// StepScopes::CurScope() get the current scope +// StepScopes::ExScope() get the ex-scope, or scope in previous time step. +// StepScopes::Next() move to next time step. +// +// if is_train = False, then +// there are two scopes for the RNN and just support forward. +// else +// the len(scopes) == seq_len +// +// if is_backward = True, then +// reversely access scopes +// else +// access scopes from begin to end. +class StepScopes { + public: + StepScopes(const framework::Scope &parent, StepScopeVar *scopes, + bool is_train, size_t seq_len, bool is_backward = false) + : counter_(is_backward ? seq_len - 1 : 0UL), + scopes_(scopes), + is_train_(is_train), + is_backward_(is_backward) { + size_t num_step_scopes = is_train ? seq_len : 2; + PADDLE_ENFORCE(is_train || !is_backward, + "Cannot backward when is not training"); + if (!is_backward_) { + PADDLE_ENFORCE(scopes->empty()); + scopes->reserve(static_cast(num_step_scopes)); + for (size_t i = 0; i < num_step_scopes; ++i) { + scopes->emplace_back(&parent.NewScope()); + } } - (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx); -} - -void RecurrentAlgorithm::CreateScopes(const Scope& scope, - size_t seq_len) const { - // TODO(superjom) Only two scopes are needed for inference, this case will be - // supported later. - auto* step_scopes_var = scope.FindVar(arg_->step_scopes); - PADDLE_ENFORCE(step_scopes_var != nullptr, ""); - auto* step_scopes = step_scopes_var->GetMutable>(); - - // Now all variables in scope must be created outside of op. - PADDLE_ENFORCE_NOT_NULL(stepnet_); - PADDLE_ENFORCE(!(*stepnet_)->Outputs().empty(), - "step_unit_ op has no outputs"); - - if (seq_len > step_scopes->size()) { - for (size_t i = step_scopes->size(); i < seq_len; ++i) { - auto& step_scope = scope.NewScope(); - - // create step net's temp inputs - for (auto& input : (*stepnet_)->Inputs()) { - // the weight are located in parent scope - for (auto& var_name : input.second) { - if (!step_scope.FindVar(var_name)) { - step_scope.Var(var_name)->GetMutable(); - } + } + + framework::Scope &CurScope() { return GetScope(counter_); } + + framework::Scope &ExScope() { + auto &scope = GetScope(is_backward_ ? counter_ + 1 : counter_ - 1); + return scope; + } + + void Next() { + if (is_backward_) { + --counter_; + } else { + ++counter_; + } + } + + private: + framework::Scope &GetScope(size_t scope_id) const { + if (!is_train_) { + scope_id %= 2; + } + PADDLE_ENFORCE_LT(scope_id, scopes_->size()); + return *(*scopes_)[scope_id]; + } + + size_t counter_; + StepScopeVar *scopes_; + bool is_train_; + bool is_backward_; +}; + +// Base class for RecurrentOp/RecurrentGradOp +// Some common protected functions for RecurrentOp/RecurrentGradOp +class RecurrentBase : public framework::OperatorBase { + public: + RecurrentBase(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + // Get SequenceLength from Scope + // The sequence length is got from input tensor. The input tensor's + // dimension should be [SEQ_LEN, ..., ...]. The first of the tensor's shape + // is SEQ_LEN. The second of the tensor's shape could be the batch size or + // nested sequence length. + int64_t GetSequenceLength(const framework::Scope &scope) const { + // Dim format SEQ_LEN, BATCH_SIZE, ... + int64_t seq_len = -1; + auto &all_inputs = Inputs(kInputs); + PADDLE_ENFORCE(!all_inputs.empty()); + for (auto &iname : all_inputs) { + auto *var = scope.FindVar(iname); + PADDLE_ENFORCE(var != nullptr); + PADDLE_ENFORCE(var->IsType()); + auto &dim = var->Get().dims(); + if (seq_len == -1) { + seq_len = dim[0]; + } else { + PADDLE_ENFORCE_EQ(seq_len, dim[0]); + } + } + return seq_len; + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.Var, dst_vars)): + // dst_tensor.ShareDataWith(src_tensor) + static void LinkTensor(const framework::Scope &src_scope, + const std::vector &src_vars, + framework::Scope *dst_scope, + const std::vector &dst_vars) { + LinkTensorWithCallback( + src_scope, src_vars, dst_scope, dst_vars, + [&](const framework::Tensor &src, framework::Tensor *dst) { + dst->ShareDataWith(src); + }); + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.Var, dst_vars)): + // callback(src_tensor, &dst_tensor) + template + static void LinkTensorWithCallback(const framework::Scope &src_scope, + const std::vector &src_vars, + framework::Scope *dst_scope, + const std::vector &dst_vars, + Callback callback) { + PADDLE_ENFORCE_EQ(src_vars.size(), dst_vars.size()); + for (size_t i = 0; i < dst_vars.size(); ++i) { + VLOG(10) << "Link " << src_vars[i] << " to " << dst_vars[i]; + AccessTensor(src_scope, src_vars[i], dst_scope, dst_vars[i], callback); + } + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.FindVar, dst_vars)): + // callback(src_tensor, &dst_tensor) + template + static void LinkTensorWithCallback(const framework::Scope &src_scope, + const std::vector &src_vars, + const framework::Scope &dst_scope, + const std::vector &dst_vars, + Callback callback) { + PADDLE_ENFORCE_EQ(src_vars.size(), dst_vars.size()); + for (size_t i = 0; i < dst_vars.size(); ++i) { + VLOG(10) << "Link " << src_vars[i] << " to " << dst_vars[i]; + AccessTensor(src_scope, src_vars[i], dst_scope, dst_vars[i], callback); + } + } + + // (seq_len, shape) -> return [seq_len] + list(shape) + static framework::DDim PrependDims(size_t seq_len, + const framework::DDim &src) { + auto dims = framework::vectorize(src); + dims.insert(dims.begin(), static_cast(seq_len)); + return framework::make_ddim(dims); + } + + private: + template + static void AccessTensor(const framework::Scope &src_scope, + const std::string &src_var_name, + framework::Scope *dst_scope, + const std::string &dst_var_name, Callback callback) { + auto *src_var = src_scope.FindVar(src_var_name); + PADDLE_ENFORCE(src_var != nullptr); + auto &src_tensor = src_var->Get(); + + auto *dst_var = dst_scope->Var(dst_var_name); + auto *dst_tensor = dst_var->GetMutable(); + callback(src_tensor, dst_tensor); + } + + template + static void AccessTensor(const framework::Scope &src_scope, + const std::string &src_var_name, + const framework::Scope &dst_scope, + const std::string &dst_var_name, Callback callback) { + auto *src_var = src_scope.FindVar(src_var_name); + PADDLE_ENFORCE(src_var != nullptr); + auto &src_tensor = src_var->Get(); + auto *dst_var = dst_scope.FindVar(dst_var_name); + PADDLE_ENFORCE(dst_var != nullptr); + auto *dst_tensor = dst_var->GetMutable(); + callback(src_tensor, dst_tensor); + } +}; + +class RecurrentOp : public RecurrentBase { + public: + RecurrentOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : RecurrentBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto seq_len = static_cast(this->GetSequenceLength(scope)); + VLOG(3) << "Static RNN input sequence length = " << seq_len; + StepScopes scopes = CreateStepScopes(scope, seq_len); + auto reverse = Attr(kReverse); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + for (size_t i = 0; i < seq_len; ++i) { + size_t seq_offset = reverse ? seq_len - i - 1 : i; + VLOG(3) << "Recurrent operate at the time step " << seq_offset; + + auto &cur_scope = scopes.CurScope(); + + // Link outside::input --> inside::input + // inside::input = outside::input[seq_offset: seq_offset+1] + LinkTensorWithCallback( + scope, Inputs(kInputs), &cur_scope, Inputs(kInputs), + [&seq_offset](const framework::Tensor &outside, + framework::Tensor *inside) { + inside->ShareDataWith(outside.Slice(seq_offset, seq_offset + 1)); + auto dims = framework::vectorize(inside->dims()); + dims.erase(dims.begin()); + inside->Resize(framework::make_ddim(dims)); + }); + + if (i == 0) { + // Link initial states --> ex_states + LinkTensor(scope, Inputs(kInitialStates), &cur_scope, + Attr>(kExStates)); + } else { + auto &ex_scope = scopes.ExScope(); + // Link ex_scope::state --> cur_scope::ex_state + LinkTensor(ex_scope, Attr>(kStates), + &cur_scope, Attr>(kExStates)); + } + + // Every inputs are linked now, execute! + executor.Run(*program, &cur_scope, block->ID(), + false /*create_local_scope*/); + + // Copy inside::output -> outside::output + // outside::output[seq_offset: seq_offset + 1] = inside::output + this->LinkTensorWithCallback( + cur_scope, Outputs(kOutputs), scope, Outputs(kOutputs), + [&](const framework::LoDTensor &src_tensor, + framework::LoDTensor *dst_tensor) { + if (i == 0) { // create output tensor at begin + dst_tensor->Resize(PrependDims(seq_len, src_tensor.dims())); + dst_tensor->mutable_data(dev_ctx.GetPlace(), src_tensor.type()); + } + + auto dst_out = dst_tensor->Slice(seq_offset, seq_offset + 1); + // Explicit copy output since the local RNN scope can be destroyed + // early. + dst_out.CopyFrom(src_tensor, dev_ctx.GetPlace(), dev_ctx); + }); + + scopes.Next(); + } + } + + private: + StepScopes CreateStepScopes(const framework::Scope &scope, + size_t seq_len) const { + auto *var = scope.FindVar(Output(kStepScopes)); + PADDLE_ENFORCE(var != nullptr); + return StepScopes(scope, var->GetMutable(), + Attr(kIsTrain), seq_len); + } +}; + +class RecurrentGradOp : public RecurrentBase { + public: + RecurrentGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : RecurrentBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto seq_len = static_cast(GetSequenceLength(scope)); + StepScopes scopes = CreateStepScopes(scope, seq_len); + auto reverse = Attr(kReverse); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + for (size_t step_id = 0; step_id < seq_len; ++step_id) { + size_t seq_offset = reverse ? step_id : seq_len - step_id - 1; + VLOG(3) << "Recurrent backward operate at the time step " << seq_offset; + auto &cur_scope = scopes.CurScope(); + // Link outside::output_grads --> inside::output_grads + // inside::output_grad = outside::output_grad[seq_offset:seq_offset+1] + LinkTensorWithCallback( + scope, Inputs(kOutputGrads), &cur_scope, Inputs(kOutputGrads), + [&](const framework::Tensor &outside, framework::Tensor *inside) { + inside->ShareDataWith(outside.Slice(seq_offset, seq_offset + 1)); + auto dims = framework::vectorize(inside->dims()); + dims.erase(dims.begin()); + inside->Resize(framework::make_ddim(dims)); + }); + auto og_set = List2Set(Inputs(kOutputGrads)); + + if (VLOG_IS_ON(10)) { + std::ostringstream sout; + std::copy(og_set.begin(), og_set.end(), + std::ostream_iterator(sout, ",")); + VLOG(10) << " RNN output gradients = [" << sout.str() << "]"; + } + + // Link states + // if cur_scope::cur_state_grad in out_grads: + // cur_scope::cur_state_grad += ex_scope::ex_state_grad + // else: + // ex_scope::ex_state_grad --> cur_scope::cur_state_grad + if (step_id != 0) { // not at beginning + auto &ex_scope = scopes.ExScope(); + auto ex_state_grads = + GradVarLists(Attr>(kExStates)); + auto cur_state_grads = + GradVarLists(Attr>(kStates)); + + PADDLE_ENFORCE_EQ(ex_state_grads.size(), cur_state_grads.size()); + for (size_t i = 0; i < ex_state_grads.size(); ++i) { + auto &cur_grad = cur_state_grads[i]; + auto &ex_grad = ex_state_grads[i]; + auto &ex_tensor = + ex_scope.FindVar(ex_grad)->Get(); + + VLOG(10) << " RNN link " << cur_grad << " from " << ex_grad; + auto *cur_grad_var = cur_scope.Var(cur_grad); + auto cur_grad_tensor = + cur_grad_var->GetMutable(); + cur_grad_tensor->CopyFrom(ex_tensor, dev_ctx.GetPlace(), dev_ctx); } } - // create stepnet's outputs - for (const auto& output : (*stepnet_)->Outputs()) { - for (auto& var_name : output.second) { - step_scope.Var(var_name); + + VLOG(5) << "Recurrent memory linking finished "; + // Run step block with cur_scope + executor.Run(*program, &cur_scope, block->ID(), + false /*create_local_scope*/); + + VLOG(5) << "executor.Run finished "; + + auto local_var_names = LocalVarNames(cur_scope); + + // Accumulate params + // if (step == 0): + // outside::param_grad = 0.0 + // outside::param_grad += inside::param_grad + { + auto &pg_names = Outputs(kParamGrads); + auto &p_names = Inputs(kParameters); + PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); + + for (size_t prog_id = 0; prog_id < pg_names.size(); ++prog_id) { + auto inside_grad_name = framework::GradVarName(p_names[prog_id]); + + // If does not compute gradient of that variable inside rnn, just + // continue + if (local_var_names.find(inside_grad_name) == local_var_names.end()) { + continue; + } + + // zero gradient variable in step 0 + if (step_id == 0) { + auto &inside_tensor = cur_scope.FindVar(inside_grad_name) + ->Get(); + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(inside_tensor.type()); + attrs["shape"] = framework::vectorize2int(inside_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {pg_names[prog_id]}}}, attrs); + zero_op->Run(scope, dev_ctx); + } + + // sum gradient + auto *outside_var = scope.FindVar(pg_names[prog_id]); + PADDLE_ENFORCE(outside_var != nullptr); + auto &outside_tensor = + *outside_var->GetMutable(); + + std::string result_var_name; + auto *local_result_var = cur_scope.Var(&result_var_name); + auto &local_result_tensor = + *local_result_var->GetMutable(); + + local_result_tensor.ShareDataWith(outside_tensor); + + auto sum_op = framework::OpRegistry::CreateOp( + "sum", {{"X", {result_var_name, inside_grad_name}}}, + {{"Out", {result_var_name}}}, {}); + sum_op->Run(cur_scope, dev_ctx); } } - step_scopes->emplace_back(&step_scope); + VLOG(5) << "Accumulate Parameter finished "; + + // Copy input gradient from inside to outside + // outside::input_grad[seq_offset: seq_offset + 1] = inside::input_grad + LinkTensorWithCallback( + cur_scope, GradVarLists(Inputs(kInputs)), scope, Outputs(kInputGrads), + [&](const framework::LoDTensor &inside, + framework::LoDTensor *outside) { + if (inside.memory_size() == 0) { // IG is not created. + return; + } + if (step_id == 0) { // alloc memory + outside->Resize(PrependDims(seq_len, inside.dims())); + outside->mutable_data(dev_ctx.GetPlace(), inside.type()); + } + + auto dst = outside->Slice(seq_offset, seq_offset + 1); + dst.CopyFrom(inside, dev_ctx.GetPlace(), dev_ctx); + }); + VLOG(5) << "Link outside gradient finished "; + + if (step_id + 1 == seq_len) { // at_end + // copy initialize states gradient from inside to outside + LinkTensorWithCallback( + cur_scope, GradVarLists(Attr>(kExStates)), + scope, Outputs(kInitStateGrads), + [&](const framework::LoDTensor &inside, + framework::LoDTensor *outside) { + outside->Resize(inside.dims()); + outside->mutable_data(dev_ctx.GetPlace(), inside.type()); + outside->CopyFrom(inside, dev_ctx.GetPlace(), dev_ctx); + }); + VLOG(5) << "Link initialize state gradient finished "; + } + scopes.Next(); } } -} - -void RecurrentAlgorithm::InitMemories(Scope* step_scope) const { - for (auto& attr : arg_->states) { - auto* pre_mem = step_scope->Var(attr.pre_var)->GetMutable(); - PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr, - "memory [%s]'s boot variable [%s] not exists", attr.var, - attr.boot_var); - auto* boot_mem = - step_scope->FindVar(attr.boot_var)->GetMutable(); - pre_mem->Resize(boot_mem->dims()); - PADDLE_ENFORCE_EQ(pre_mem->dims().size(), 2); - pre_mem->ShareDataWith(*boot_mem); - } -} - -const rnn::ArgumentName RecurrentOp::kArgName{ - "step_net", "step_scopes", "inputs", "outputs", - "states", "ex_states", "initial_states"}; - -const rnn::ArgumentName RecurrentGradientOp::kArgName{ - "step_net", "step_scopes@GRAD", "outputs@GRAD", "inputs@GRAD", - "states", "ex_states", "initial_states@GRAD"}; - -RecurrentOp::RecurrentOp(const std::string& type, - const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs) - : OperatorBase(type, inputs, outputs, attrs) { - rnn::InitArgument(kArgName, &arg_, *this); - alg_.Init(&arg_, &stepnet_); -} - -class RecurrentAlgorithmProtoAndCheckerMaker - : public framework::OpProtoAndCheckerMaker { + + private: + StepScopes CreateStepScopes(const framework::Scope &scope, + size_t seq_len) const { + auto *var = scope.FindVar(Input(kStepScopes)); + PADDLE_ENFORCE(var != nullptr); + return StepScopes(scope, var->GetMutable(), + Attr(kIsTrain), seq_len, true /*is_backward*/); + } + + std::unordered_set List2Set( + const std::vector &list) const { + std::unordered_set local_var_name_set; + local_var_name_set.reserve(list.size()); + for (auto &each : list) { + local_var_name_set.insert(each); + } + return local_var_name_set; + } + + std::unordered_set LocalVarNames( + const framework::Scope &scope) const { + return this->List2Set(scope.GetAllNames(false)); + } + static std::vector GradVarLists( + const std::vector &var_names) { + std::vector retv; + retv.reserve(var_names.size()); + std::transform(var_names.begin(), var_names.end(), std::back_inserter(retv), + framework::GradVarName); + return retv; + } +}; + +class RecurrentOpProtoMaker : public framework::OpProtoAndCheckerMaker { public: - RecurrentAlgorithmProtoAndCheckerMaker(framework::OpProto* proto, - framework::OpAttrChecker* op_checker) + RecurrentOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - const auto& name = RecurrentOp::kArgName; - // inputs and outputs stored in proto - AddInput(name.inlinks, - "the inputs that need to be segmented for each step.") + AddInput(kInputs, "rnn inputs").AsDuplicable(); + AddInput(kInitialStates, "rnn initial states").AsDuplicable(); + AddInput(kParameters, + "Parameters are used by step block as its input. However, the " + "input is not a sequence tensor. Every time step, each operator " + "in step block just use the parameter directly.") .AsDuplicable(); - AddInput(name.initial_states, "variables to initialize states.") + AddOutput(kOutputs, + "The output sequence of RNN. The sequence length must be same.") .AsDuplicable(); + AddOutput(kStepScopes, + "StepScopes contain all local variables in each time step."); + AddAttr>(kExStates, + string::Sprintf( + R"DOC(The ex-state variable names. +The ex-state means the state value in the ex-timestep or the previous time step +[%s, %s, %s] must be the same order)DOC", + kExStates, kStates, kInitStateGrads)); + AddAttr>( + kStates, + string::Sprintf( + "The state variable names. [%s, %s, %s] must be the same order", + kExStates, kStates, kInitStateGrads)); + AddAttr(kStepBlock, + "The step block inside RNN"); + AddAttr(kReverse, R"DOC(Calculate RNN reversely or not. +By default reverse=False - AddOutput(name.outlinks, "the outputs that need to concated for all steps.") - .AsDuplicable(); - AddOutput(name.step_scopes, "step scopes"); +Assume the input data is [A, B, C, D] + +if reverse is False: + the computation of RNN is like + A B C D + | | | | + v v v v + rnn -----> rnn -----> rnn ----> rnn + | | | | + v v v v + o o o o + +if reverse is True + the computation of RNN is like + A B C D + | | | | + v v v v + rnn <----- rnn <----- rnn <---- rnn + | | | | + v v v v + o o o o +)DOC").SetDefault(false); + AddAttr(kIsTrain, "").SetDefault(true); + AddComment(R"DOC( +Static Length Recurrent Operator. + +The static length recurrent operator can only operate on fixed size sequence +data, i.e. in each mini-batch, the sequence length of all inputs are the same. + +)DOC"); + } +}; + +class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - // Attributes stored in AttributeMap - AddAttr>(name.ex_states, "names of pre-states"); - AddAttr>(name.states, "names of states"); + protected: + virtual std::unique_ptr Apply() const { + auto *grad = new framework::OpDescBind(); + grad->SetType("recurrent_grad"); + for (auto &input_param : this->InputNames()) { + grad->SetInput(input_param, this->Input(input_param)); + grad->SetOutput(framework::GradVarName(input_param), + this->InputGrad(input_param)); + } + + for (auto &output_param : this->OutputNames()) { + if (output_param == kStepScopes) { + grad->SetInput(output_param, this->Output(output_param)); + grad->SetInput(framework::GradVarName(output_param), + this->Output(output_param)); + } else { + grad->SetInput(output_param, this->Output(output_param)); + grad->SetInput(framework::GradVarName(output_param), + this->OutputGrad(output_param)); + } + } + grad->SetAttrMap(this->Attrs()); + grad->SetBlockAttr(kStepBlock, *grad_block_[0]); - AddComment("This is a recurrent group operator."); + return std::unique_ptr(grad); } }; -void RecurrentGradientAlgorithm::Run( - const Scope& scope, const platform::DeviceContext& dev_ctx) const { - auto* input0 = scope.FindVar(arg_->inlinks[0]); - PADDLE_ENFORCE_NOT_NULL(input0); - size_t seq_len = input0->GetMutable()->dims()[0]; - auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); - for (int step_id = seq_len - 1; step_id >= 0; --step_id) { - if (static_cast(step_id) != seq_len - 1) { - rnn::LinkMemories(step_scopes, arg_->states, step_id, 1); +class RecurrentGradOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + std::vector input{kInputs, kInitialStates}; + std::vector output{kOutputs}; + for (auto &s : input) { + PADDLE_ENFORCE(ctx->HasInputs(s)); + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(s))); + } + for (auto &s : output) { + PADDLE_ENFORCE(ctx->HasInputs(s)); + } + for (auto &s : input) { + ctx->SetOutputsDim(framework::GradVarName(s), ctx->GetInputsDim(s)); } - (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx); - LinkBootMemoryGradients(step_scopes[0]); -} - -void RecurrentGradientAlgorithm::LinkBootMemoryGradients( - Scope* step_scope) const { - for (auto& attr : arg_->states) { - PADDLE_ENFORCE(step_scope->FindVar(attr.var) != nullptr, - "memory variable [%s] does not exists", attr.var); - PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr, - "boot variable [%s] does not exists", attr.boot_var); - auto* mem_grad = step_scope->Var(attr.var)->GetMutable(); - auto* boot_mem_grad = - step_scope->Var(attr.boot_var)->GetMutable(); - boot_mem_grad->Resize(mem_grad->dims()); - boot_mem_grad->ShareDataWith(*mem_grad); - } -} - -RecurrentGradientOp::RecurrentGradientOp( - const std::string& type, const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs) - : OperatorBase(type, inputs, outputs, attrs) { - rnn::InitArgument(kArgName, &arg_, *this, true /*is grad*/); - alg_.Init(&arg_, &stepnet_); -} + if (ctx->HasInputs(kParameters)) { + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters))); + ctx->SetOutputsDim(framework::GradVarName(kParameters), + ctx->GetInputsDim(kParameters)); + } + } +}; } // namespace operators } // namespace paddle -REGISTER_OP(recurrent, paddle::operators::RecurrentOp, - paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker, - recurrent_grad, paddle::operators::RecurrentGradientOp); +REGISTER_OPERATOR(recurrent, paddle::operators::RecurrentOp, + paddle::operators::RecurrentOpProtoMaker, + paddle::operators::RecurrentGradOpDescMaker); +REGISTER_OPERATOR(recurrent_grad, paddle::operators::RecurrentGradOp, + paddle::operators::RecurrentGradOpShapeInference); diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h deleted file mode 100644 index 253d7e3284360ceaddce9ef5f8f9a3ea4793d740..0000000000000000000000000000000000000000 --- a/paddle/operators/recurrent_op.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once - -#include "paddle/framework/operator.h" -#include "paddle/operators/net_op.h" -#include "paddle/operators/rnn/recurrent_op_utils.h" - -namespace paddle { -namespace operators { - -// The sequence format in RecurrentOp is Tensor now. -// TODO(Superjom) -// 1. No-padding computing for sequences with indifinite length in one batch. -// 2. Hierarchical RNN for sequence with sub-sequence. -// 3. Internal Memory. -// 4. More Complex RNN architecture, such as Gated Feedback RNN. -// Refer to: https://arxiv.org/pdf/1502.02367.pdf - -class RecurrentAlgorithm { - public: - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const; - - void Init(rnn::Argument* arg, - std::unique_ptr* stepnet) { - PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before."); - arg_ = arg; - stepnet_ = stepnet; - } - - protected: - /* - * The step scopes will be stored in the father scope as a variable. - * - * NOTE the scopes are reused in both the forward and backward, so just - * create once and expand its size if more steps need. - */ - void CreateScopes(const framework::Scope& scope, size_t seq_len) const; - - const std::vector& GetStepScopes( - const framework::Scope& scope) const { - return *scope.FindVar(arg_->step_scopes) - ->GetMutable>(); - } - - void InitMemories(framework::Scope* step_scopes) const; - - private: - std::unique_ptr* stepnet_; - rnn::Argument* arg_; -}; - -class RecurrentGradientAlgorithm { - /** - * RNN's backward alogorithm. - * - * To accelerate the development of RecurrentGradientOp, we decouple RNN's - * algorithm and `OperatorBase`'s implementation, the former contains the core - * implementation of a RNN, and will keep stable even if the framework changes - * a - * lot, and the latter is a wrapper acts like an dapter for it to make RNN an - * operator. - */ - public: - void Init(rnn::Argument* arg, - std::unique_ptr* stepnet) { - PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before."); - arg_ = std::move(arg); - stepnet_ = stepnet; - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const; - - void LinkBootMemoryGradients(framework::Scope* step_scopes) const; - - protected: - inline const std::vector& GetStepScopes( - const framework::Scope& scope) const { - return *scope.FindVar(arg_->step_scopes) - ->GetMutable>(); - } - - private: - rnn::Argument* arg_; - std::unique_ptr* stepnet_; -}; - -class RecurrentOp : public framework::OperatorBase { - public: - RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs); - - RecurrentOp(const RecurrentOp& o) - : framework::OperatorBase( - static_cast(o)) { - // TODO(yuyang18): Implement copy ctor well. - PADDLE_THROW("Not implemented"); - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const override { - alg_.Run(scope, dev_ctx); - } - - void set_stepnet(std::unique_ptr net) { - stepnet_ = std::move(net); - } - - const OperatorBase& stepnet() const { return *stepnet_; } - - static const rnn::ArgumentName kArgName; - - private: - RecurrentAlgorithm alg_; - rnn::Argument arg_; - std::unique_ptr stepnet_; -}; - -class RecurrentGradientOp : public framework::OperatorBase { - public: - RecurrentGradientOp(const std::string& type, - const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs); - - RecurrentGradientOp(const RecurrentGradientOp& o) - : framework::OperatorBase( - static_cast(o)) { - // TODO(yuyang18): Implement Copy ctor. - PADDLE_THROW("Not Implemented"); - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const override { - alg_.Run(scope, dev_ctx); - } - - static const rnn::ArgumentName kArgName; - - /* - * set a stepnet that is created according to a RecurrentOp's stepnet. - */ - void set_stepnet(std::unique_ptr net) { - stepnet_ = std::move(net); - } - const OperatorBase& stepnet() const { return *stepnet_; } - - private: - RecurrentGradientAlgorithm alg_; - std::unique_ptr stepnet_; - rnn::Argument arg_; -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc index 0599daa7688a5658ebea8902c4e15e63570539fb..2589a54cfc7fc5bc11ae983797d480a134e0eb25 100644 --- a/paddle/operators/reduce_op.cc +++ b/paddle/operators/reduce_op.cc @@ -80,24 +80,27 @@ class ReduceOpMaker : public framework::OpProtoAndCheckerMaker { public: ReduceOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor) The input tensor. Tensors with rank at most 6 are supported"); + AddInput("X", + "(Tensor) The input tensor. Tensors with rank at most 6 are " + "supported."); AddOutput("Out", "(Tensor) The result tensor."); AddAttr( "dim", - "(int, default 1) The dimension to reduce. " + "(int, default 0) The dimension to reduce. " "Must be in the range [-rank(input), rank(input)). " "If `dim < 0`, the dim to reduce is `rank + dim`. " - "Noting that reducing on the first dim will make the LoD info lost.") + "Note that reducing on the first dim will make the LoD info lost.") .SetDefault(0); AddAttr("keep_dim", "(bool, default false) " "If true, retain the reduced dimension with length 1.") .SetDefault(false); comment_ = R"DOC( -{ReduceOP} operator computes the {reduce} of input tensor along the given dimension. -The result tensor has 1 fewer dimension than the input unless `keep_dim` is true. +{ReduceOp} Operator. + +This operator computes the {reduce} of input tensor along the given dimension. +The result tensor has 1 fewer dimension than the input unless keep_dim is true. + )DOC"; AddComment(comment_); } diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index eda8226480a66ae1a631391e9335db04604039c5..ba774ec2160c0460867de42f7ad9d5cd65ad8d6a 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -36,7 +36,7 @@ class ReshapeOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(shape.size() > 0, "Attr(shape) shouldn't be empty."); auto x_dims = ctx->GetInputDim("X"); // TODO(qiao) change batch_size - for (int i = 1; i < shape.size(); ++i) { + for (size_t i = 1; i < shape.size(); ++i) { PADDLE_ENFORCE(shape[i] > 0, "Each dimension of shape " "must be positiv except the first."); @@ -71,8 +71,11 @@ class ReshapeOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input tensor of reshape operator."); AddOutput("Out", "The output tensor of reshape operator."); - AddAttr>("shape", "Target shape of reshape operator."); - AddComment(R"DOC(Reshape operator + AddAttr>("shape", + "(vector) " + "Target shape of reshape operator."); + AddComment(R"DOC( +Reshape Operator. Reshape Input(X) into the shape specified by Attr(shape). @@ -81,7 +84,7 @@ Given a 2-D tensor X with 2 rows and 2 columns [[1, 2], [3, 4]] -with target shape = [1, 4], the reshape operator will transform +and target shape = [1, 4], the reshape operator will transform the tensor X into a 1-D tensor: [1, 2, 3, 4] diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index fd5567a365c4c843de3b8aec7fa77164f16644a4..a9c45f639c6728ff2fd6de6fcdadfe5032a705d7 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -68,22 +68,22 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Param", "(Tensor, default Tensor) " - "Input parameter value that has to be updated"); + "Input parameter value that has to be updated."); AddInput("MeanSquare", "(Tensor, default Tensor)" - " The mean square value that gets updated"); + " The mean square value that gets updated."); AddInput("LearningRate", "(Tensor, default Tensor) " - "The learning rate should be a tensor of size 1"); + "The learning rate should be a tensor of size 1."); AddInput("Grad", "(Tensor, default Tensor) " - "Input gradient of the parameter"); + "Input gradient of the parameter."); AddInput("Moment", - "(Tensor, default Tensor) The moment that gets updated"); + "(Tensor, default Tensor) The moment that gets updated."); - AddOutput("ParamOut", "(Tensor) Output updated parameter value"); - AddOutput("MomentOut", "(Tensor) Output updated moment"); - AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value"); + AddOutput("ParamOut", "(Tensor) Output updated parameter value."); + AddOutput("MomentOut", "(Tensor) Output updated moment."); + AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value."); AddAttr("epsilon", "(float, default 1e-10) Constant " @@ -93,18 +93,19 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { "(float, default 0.9) " "Discounting factor for coming gradient.") .SetDefault(0.9f); - AddAttr("momentum", "(float, default 0.0) Constant value") + AddAttr("momentum", "(float, default 0.0) Constant value.") .SetDefault(0.0f); AddComment(R"DOC( +Rmsprop Optimizer. -RMSprop - -MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad +$$ +MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad \\ MomentOut = momentum * Moment + - LearningRate * Grad / sqrt(MeanSquareOut + epsilon) + \frac{LearningRate * Grad}{\sqrt{MeanSquareOut + epsilon}} \\ ParamOut = Param - MomentOut +$$ -The original slides that proposed RMSprop: Slide 29 of +The original slides that proposed Rmsprop: Slide 29 of http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) )DOC"); diff --git a/paddle/operators/rnn_memory_helper_op.cc b/paddle/operators/rnn_memory_helper_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..b621c7f1ba3f9e9613dea5bc98ef74c7c6dae9a0 --- /dev/null +++ b/paddle/operators/rnn_memory_helper_op.cc @@ -0,0 +1,153 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace operators { +class RNNMemoryHelperOp : public framework::OperatorBase { + public: + RNNMemoryHelperOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto mem_var_name = Input("X"); + auto *mem_var = scope.FindVar(mem_var_name); + PADDLE_ENFORCE(mem_var != nullptr, + "Cannot find mem_var in scope, mem_var_name is %s", + mem_var_name); + + auto out_name = this->Output("Out"); + auto *out_var = scope.FindVar(out_name); + PADDLE_ENFORCE(out_var != nullptr, + "Cannot find out_var in scope, out_var_name is %s", + out_name); + + auto *out_tensor = out_var->GetMutable(); + auto &mem_tensor = mem_var->Get(); + out_tensor->ShareDataWith(mem_tensor); + out_tensor->set_lod(mem_tensor.lod()); + } +}; + +class RNNMemoryHelperOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), ""); + PADDLE_ENFORCE(ctx->HasOutput("Out"), ""); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class RNNMemoryHelperOpInfoMaker : public framework::OpProtoAndCheckerMaker { + public: + RNNMemoryHelperOpInfoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddOutput("Out", ""); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") + .SetDefault(framework::DataType::FP32); + AddComment(""); + } +}; + +class RNNMemoryHelperGradOp : public framework::OperatorBase { + public: + RNNMemoryHelperGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto out_grad_var_name = Input(framework::GradVarName("Out")); + auto *out_grad_var = scope.FindVar(out_grad_var_name); + + auto in_grad_var_name = Output(framework::GradVarName("X")); + auto *in_grad_var = scope.FindVar(in_grad_var_name); + PADDLE_ENFORCE(in_grad_var != nullptr, + "Cannot find in_grad_var in scope, name is %s", + in_grad_var_name); + + if (out_grad_var == nullptr) { + VLOG(5) << "Using fill constant 0 as starting gradient"; + auto in_var_name = Input("X"); + auto *in_var = scope.FindVar(in_var_name); + auto &in_var_tensor = in_var->Get(); + + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(in_var_tensor.type()); + attrs["shape"] = framework::vectorize2int(in_var_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {in_grad_var_name}}}, attrs); + zero_op->Run(scope, dev_ctx); + } else { + auto &out_grad_tensor = out_grad_var->Get(); + auto *in_grad_tensor = in_grad_var->GetMutable(); + in_grad_tensor->ShareDataWith(out_grad_tensor); + in_grad_tensor->set_lod(out_grad_tensor.lod()); + } + } +}; + +class RNNMemoryHelperGradOpInfoMaker + : public framework::OpProtoAndCheckerMaker { + public: + RNNMemoryHelperGradOpInfoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput(framework::GradVarName("Out"), ""); + AddInput("X", ""); + AddInput("Out", ""); + AddOutput(framework::GradVarName("X"), ""); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") + .SetDefault(framework::DataType::FP32); + AddComment(""); + } +}; + +class RNNMemoryHelperGradOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + auto x_grad_name = framework::GradVarName("X"); + PADDLE_ENFORCE(ctx->HasOutput(x_grad_name), ""); + PADDLE_ENFORCE(ctx->HasInput("X"), ""); + ctx->SetOutputDim(x_grad_name, ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ x_grad_name); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(rnn_memory_helper, paddle::operators::RNNMemoryHelperOp, + paddle::operators::RNNMemoryHelperOpInfoMaker, + paddle::operators::RNNMemoryHelperOpShapeInference, + paddle::framework::DefaultGradOpDescMaker); +REGISTER_OPERATOR(rnn_memory_helper_grad, + paddle::operators::RNNMemoryHelperGradOp, + paddle::operators::RNNMemoryHelperGradOpInfoMaker, + paddle::operators::RNNMemoryHelperGradOpShapeInference); diff --git a/paddle/operators/save_load_op_test.cc b/paddle/operators/save_load_op_test.cc index fe2b15ec09c6d29ad5f78e5c36f534c6a88497e6..a57466a48d4d6016fe2618d19fdca4c4f667124a 100644 --- a/paddle/operators/save_load_op_test.cc +++ b/paddle/operators/save_load_op_test.cc @@ -34,7 +34,7 @@ TEST(SaveLoadOp, CPU) { tensor->set_lod(expect_lod); int* expect = tensor->mutable_data(place); - for (size_t i = 0; i < paddle::framework::product(tensor->dims()); ++i) { + for (int64_t i = 0; i < tensor->numel(); ++i) { expect[i] = static_cast(i); } paddle::framework::AttributeMap attrs; @@ -50,7 +50,7 @@ TEST(SaveLoadOp, CPU) { "load", {}, {{"Out", {"out_var"}}}, attrs); load_op->Run(scope, ctx); int* actual = target->data(); - for (size_t i = 0; i < paddle::framework::product(tensor->dims()); ++i) { + for (int64_t i = 0; i < tensor->numel(); ++i) { EXPECT_EQ(expect[i], actual[i]); } auto& actual_lod = target->lod(); @@ -60,4 +60,4 @@ TEST(SaveLoadOp, CPU) { EXPECT_EQ(expect_lod[i][j], actual_lod[i][j]); } } -} \ No newline at end of file +} diff --git a/paddle/operators/save_op.cc b/paddle/operators/save_op.cc index 490256dfa1cf9b891713dac264e9260906ce1025..56909fb65f44ad00314103e21bee9535fbd59317 100644 --- a/paddle/operators/save_op.cc +++ b/paddle/operators/save_op.cc @@ -163,14 +163,19 @@ class SaveOpProtoMaker : public framework::OpProtoAndCheckerMaker { SaveOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The tensor need to be saved"); - AddComment(R"DOC(Save operator -Save operator will serialize and write a tensor variable to disk file. + AddInput("X", "(Tensor ) Input tensor to be saved"); + AddComment(R"DOC( +Save operator + +This operator will serialize and write a tensor variable to file on disk. )DOC"); - AddAttr("overwrite", "Overwrite the output file if exist") + AddAttr("overwrite", + "(boolean, default true)" + "Overwrite the output file if exist") .SetDefault(true); AddAttr("file_path", - "Variable will be saved to \"file_path\".") + "(string)" + "The \"file_path\" where the variable will be saved.") .AddCustomChecker( [](const std::string &path) { return !path.empty(); }); } diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 5fcacf70d80527b4580a8f744ab3b79fb301d1d9..5745580504fb9bda551f21665bff5c65ae82aeb9 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -40,13 +40,16 @@ class ScaleOpMaker : public framework::OpProtoAndCheckerMaker { public: ScaleOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The input tensor of scale operator."); - AddOutput("Out", "The output tensor of scale operator."); - AddComment(R"DOC(Scale operator + AddInput("X", "(Tensor) Input tensor of scale operator."); + AddOutput("Out", "(Tensor) Output tensor of scale operator."); + AddComment(R"DOC( +Scale operator -The equation is: Out = scale*X +$$Out = scale*X$$ )DOC"); - AddAttr("scale", "The scaling factor of the scale operator.") + AddAttr("scale", + "(float, default 0)" + "The scaling factor of the scale operator.") .SetDefault(1.0); } }; diff --git a/paddle/operators/seq_expand_op.cc b/paddle/operators/seq_expand_op.cc index 08fda9b44564249634f0d1a570e8b2458f88fd41..b862056ad400290a60e8a75a23dceeb1d4422ea4 100644 --- a/paddle/operators/seq_expand_op.cc +++ b/paddle/operators/seq_expand_op.cc @@ -53,8 +53,10 @@ class SeqExpandOpMaker : public framework::OpProtoAndCheckerMaker { "(LodTensor)The output of seq_expand op." "The lod of output will be as same as input(Y)'s lod."); AddComment(R"DOC( -Expand input(X) according to LOD of input(Y). +Seq Expand Operator. +This operator expands input(X) according to LOD of input(Y). +Following are cases to better explain how this works: Case 1: Given 2-level a LoDTensor input(X) diff --git a/paddle/operators/seq_expand_op.h b/paddle/operators/seq_expand_op.h index aa91e0f9296a7856f4723d413ca0de6876ab6f3b..4ef0d02cf85c43e95335660be65a67df66b4f55c 100644 --- a/paddle/operators/seq_expand_op.h +++ b/paddle/operators/seq_expand_op.h @@ -32,7 +32,8 @@ class SeqExpandKernel : public framework::OpKernel { const T* x_data = x->data(); auto x_dims = x->dims(); auto* y = context.Input("Y"); - PADDLE_ENFORCE_EQ(x_dims[0], y->lod().back().size() - 1, + PADDLE_ENFORCE_EQ(static_cast(x_dims[0]), + y->lod().back().size() - 1, "The size of last lod level in Input(Y)" "must be equal to dims[0] of Input(X)."); out->set_lod(y->lod()); @@ -48,7 +49,7 @@ class SeqExpandKernel : public framework::OpKernel { x_t(x_data, 1, element_len); Eigen::TensorMap> out_t(out_data, scale, element_len); - Eigen::array cast({scale, 1}); + Eigen::array cast({{scale, 1}}); out_t.device(place) = x_t.broadcast(cast); x_data += element_len; out_data += element_len * scale; diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc index 46f73e3c279835bbb4bfdd7dede03a5535186b24..64097ef2525d734f79f22ddd7957b3216b06ee7b 100644 --- a/paddle/operators/sequence_concat_op.cc +++ b/paddle/operators/sequence_concat_op.cc @@ -47,19 +47,19 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "(A vector of LoDTensor), the input is a vector of LoDTensor, " + "(vector) Input is a vector of LoDTensor, " "each of which is a variable-length sequence or nested sequence.") .AsDuplicable(); AddOutput("Out", - "(A LoDTensor), the variable-length output of " + "(LoDTensor), Variable-length output of " "sequence_concat Op."); AddAttr("axis", - "(int, default 0)" - "The axis which the inputs will be joined with. " + "(int, default 0) " + "The axis along which the inputs will be joined. " "If axis is 0, the inputs will be joined with LoD index.") .SetDefault(0); AddAttr("level", - "(int, default 0)" + "(int, default 0) " "The level at which the inputs will be joined. " "If the level is 0, the inputs will be joined at the nested " "sequence level. " @@ -68,34 +68,38 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { "The level should be less than the level number of inputs.") .SetDefault(0); AddComment(R"DOC( - The sequence_concat operator concatenates multiple LoDTensors. - It only supports sequence (LoD Tensor with level number is 1) - or a nested sequence (LoD tensor with level number is 2) as its input. - - Case1: - If the axis is other than 0(here, axis is 1 and level is 1), - each input should have the same LoD information and the LoD - information of the output keeps the same as the input. - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) - LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) - - - Case2: - If the axis is 0(here, leve is 0), the inputs are concatenated along - time steps, the LoD information of the output need to re-compute. - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,2,3,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,1,2,3,4,5,6,7,9}}; Dims(Out) = (9,3,4) - - - Case3: - If the axis is 0(here, level is 1). - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,3,4,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,2,5,7,9}}; Dims(Out) = (9,3,4) - - NOTE: The levels of all the inputs should be the same. +Sequence Concat Operator. + +The sequence_concat operator concatenates multiple LoDTensors. +It supports a sequence (LoD Tensor with level number is 1) +or a nested sequence (LoD tensor with level number is 2) as its input. +The following examples explain how the operator works: +- Case1: + If the axis is other than 0(here, axis is 1 and level is 1), + each input should have the same LoD information and the LoD + information of the output keeps the same as the input. + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) + LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) + +- Case2: + If the axis is 0(here, leve is 0), the inputs are concatenated along + time steps, the LoD information of the output need to re-compute. + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,3,5}, {0,1,2,3,5}}; Dims(x1) = (5,3,4) + LoD(Out) = {{0,5,9}, {0,1,2,3,4,5,6,7,9}}; Dims(Out) = (9,3,4) + +- Case3: + If the axis is 0(here, level is 1). + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,3,5}, {0,1,3,4,5}}; Dims(x1) = (5,3,4) + LoD(Out) = {{0,5,9}, {0,2,5,7,9}}; Dims(Out) = (9,3,4) + +NOTE: The levels of all the inputs should be the same. + )DOC"); } }; diff --git a/paddle/operators/sequence_conv_op.cc b/paddle/operators/sequence_conv_op.cc index 139000c561870c3bc49e01cdcb6cf4b787e64577..41cadce4c603a9c14db79e2f6b30f8664cf72a38 100644 --- a/paddle/operators/sequence_conv_op.cc +++ b/paddle/operators/sequence_conv_op.cc @@ -30,19 +30,20 @@ class SequenceConvOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of SequenceConvOp should not be null."); - int context_length = ctx->Attrs().Get("context_length"); - bool padding_trainable = ctx->Attrs().Get("padding_trainable"); - int context_start = ctx->Attrs().Get("context_start"); + int context_length = ctx->Attrs().Get("contextLength"); + int context_start = ctx->Attrs().Get("contextStart"); auto in_dims = ctx->GetInputDim("X"); auto filter_dims = ctx->GetInputDim("Filter"); + PADDLE_ENFORCE(ctx->Attrs().Get("contextStride") == 1, + "Currently, SequenceConvOp only supports contextStride=1."); PADDLE_ENFORCE(in_dims.size() == 2 && filter_dims.size() == 2, "Input(X, Filter) should be 2-D tensor."); PADDLE_ENFORCE(filter_dims[0] == context_length * in_dims[1], "Filter's height should be context_length * " - "number_of_input_features ."); + "input_hidden_size ."); - if (padding_trainable) { + if (ctx->Attrs().Get("paddingTrainable")) { PADDLE_ENFORCE( ctx->HasInput("PaddingData"), "Input(PaddingData) of SequenceConvOp should not be null."); @@ -54,7 +55,7 @@ class SequenceConvOp : public framework::OperatorWithKernel { if (context_start == 0 && context_length == 1) { PADDLE_THROW( - "If context_start is 0 and context_length is 1, padding_trainable " + "If context_start is 0 and context_length is 1, paddingTrainable " "should be false."); } PADDLE_ENFORCE(padding_dim.size() == 2, @@ -81,13 +82,14 @@ class SequenceConvGradOp : public framework::OperatorWithKernel { "Gradient of output(Out) should not be null."); PADDLE_ENFORCE(ctx->HasInput("X"), "The input(X) should not be null."); - if (ctx->Attrs().Get("padding_trainable") && + if (ctx->Attrs().Get("paddingTrainable") && ctx->HasOutput(framework::GradVarName("PaddingData"))) { ctx->SetOutputDim(framework::GradVarName("PaddingData"), ctx->GetInputDim("PaddingData")); } if (ctx->HasOutput(framework::GradVarName("X"))) { ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->ShareLoD("X", framework::GradVarName("X")); } if (ctx->HasOutput(framework::GradVarName("Filter"))) { ctx->SetOutputDim(framework::GradVarName("Filter"), @@ -103,62 +105,68 @@ class SequenceConvOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", - "(LoDTensor) the input(X) is a LodTensor, which support " + "(LoDTensor) the input(X) is a LodTensor, which supports " "variable-time length input sequence. The underlying tensor in " - "this LoDTensor is a matrix with shape (T, D), where, T is the " - "total time steps in this mini-batch, D is the input feature size."); + "this LoDTensor is a matrix with shape (T, N), where T is the " + "total time steps in this mini-batch and N is the input_hidden_size."); AddInput("PaddingData", "(Tensor, optional) the input(PaddingData) is an optional " "parameter, and it is learnable. " - "This is a tensor with shape (N, D), where N is the " - "top_pad + bottom_pad, D is the input feature size. In order to " + "This is a tensor with shape (P, N), where P is the " + "top_pad + bottom_pad, N is the input_hidden_size. In order to " "ensure the equal length of sequence before and after " "convolution, it is necessary to fill the top and bottom of each " "sequence according to context_length, context_stride and " "context_start") .AsDispensable(); - AddInput("Filter", - "(Tensor) the input(Filter) is an learnable parameter." - "This is a tensor with shape (N, D), where N is the " - "context_length, D is the output feature size."); + AddInput( + "Filter", + "(Tensor) the input(Filter) is an learnable parameter." + "This is a tensor with shape (K, M), where K is the " + "context_length * input_hidden_size, M is the output feature size."); AddOutput( "Out", "(LoDTensor) the output(Out) is a LodTensor, which support " "variable-time length output sequence. The underlying tensor in " - "this LoDTensor is a matrix with shape (T, D), where, T is the " - "total time steps in this mini-batch, D is the output feature size."); + "this LoDTensor is a matrix with shape (T, M), where, T is the " + "total time steps in this mini-batch, M is the output feature size."); - AddAttr("padding_trainable", - "(bool, default false) the padding data of SequenceConvOp " + AddAttr("paddingTrainable", + "(bool, default:false) the padding data of SequenceConvOp " "is trainable or not.") .SetDefault(false); - AddAttr("context_length", - "(int, default 3) the context_length of SequenceConvOp is the " + AddAttr("contextLength", + "(int) the contextLength of SequenceConvOp is the " "height of the convolution kernel.") - .SetDefault(3) .GreaterThan(0); - AddAttr("context_start", - "(int, default 0) the context_start of SequenceConvOp " + AddAttr("contextStart", + "(int, default:0) the contextStart of SequenceConvOp " "represents the beginning of the convolution of the number of " - "rows of sequence, which can be negative.") + "rows of sequence, which can be negative. The negative number " + "means to pad contextStart time-steps of zeros or learnable " + "parameters at the beginning of each instance. The positive " + "number means to skip contextStart time-steps of each " + "instance.") .SetDefault(0); - AddAttr("context_stride", - "(int, default 1) the context_stride of SequenceConvOp " - "represents the step length of convolution. " + AddAttr("contextStride", + "(int, default:1) the contextStride of SequenceConvOp " + "represents the stride length of convolution kernel. " "Currently, SequenceConvOp only supports" - "context_stride=1.") + "contextStride=1.") .SetDefault(1) .GreaterThan(0); AddComment(R"DOC( - SequenceConvOp performs convolution operation on features of - context_length time-steps of each instance. - The convolution operation calculates the output based on the input, filter - and strides, paddings parameters. The size of each dimension of the - parameters is checked in the infer-shape. In order to ensure the equal - length of sequence before and after convolution, it is necessary to fill - the top and bottom of each sequence according to context_length, - context_stride and context_start. +Sequence Conv Operator. + +SequenceConvOp performs convolution operation on features of contextLength +time-steps of each instance. The convolution operation calculates the output +based on the input, filter, strides and paddings parameters. +The size of each dimension of the parameters is checked during infer-shape. +In order to ensure the equal length of sequence before and after convolution, +it is necessary to fill the top and bottom of each sequence based on +context_length, context_stride and context_start. + )DOC"); } }; diff --git a/paddle/operators/sequence_conv_op.h b/paddle/operators/sequence_conv_op.h index cd8a8d4cea39161029602530cc75532b5f977d01..a57e1752bb8ed4844423f752bf0ad9f8e114486a 100644 --- a/paddle/operators/sequence_conv_op.h +++ b/paddle/operators/sequence_conv_op.h @@ -35,12 +35,11 @@ class SequenceConvKernel : public framework::OpKernel { out->mutable_data(context.GetPlace()); context.ShareLoD("X", "Out"); - int context_start = context.Attr("context_start"); - int context_length = context.Attr("context_length"); - int context_stride = context.Attr("context_stride"); - bool padding_trainable = context.Attr("padding_trainable"); + int context_start = context.Attr("contextStart"); + int context_length = context.Attr("contextLength"); + int context_stride = context.Attr("contextStride"); + bool padding_trainable = context.Attr("paddingTrainable"); - // InferShape by in_lod PADDLE_ENFORCE_EQ(in->lod().size(), 1UL, "Only support one level sequence now."); @@ -51,26 +50,21 @@ class SequenceConvKernel : public framework::OpKernel { int up_pad = std::max(0, -context_start); int down_pad = std::max(0, context_start + context_length - 1); - int sequence_width; - sequence_width = static_cast(in->dims()[1]); + int sequence_width = static_cast(in->dims()[1]); - // Use col_shape in the im2col calculation. framework::DDim col_shape = {in->dims()[0], - sequence_width * context_length}; + context_length * sequence_width}; Tensor col; col.mutable_data(col_shape, context.GetPlace()); - math::SetConstant set_zero; // Because if padding_trainable is false, padding data should be zeros. + math::SetConstant set_zero; set_zero(context.device_context(), &col, static_cast(0)); - paddle::operators::math::ContextProjectFunctor - seq_project_functor; - LoDTensor* input = const_cast(in); - Tensor* pad_data = const_cast(padding_data); + math::ContextProjectFunctor seq_project_functor; - seq_project_functor(context.device_context(), *input, *pad_data, col, + seq_project_functor(context.device_context(), *in, *padding_data, col, padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, false, false, false); + context_stride, up_pad, down_pad); math::matmul(context.device_context(), col, false, filter, false, static_cast(1.0), out, static_cast(0.0)); @@ -81,18 +75,18 @@ template class SequenceConvGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* out_g = context.Input(framework::GradVarName("Out")); auto* in_g = context.Output(framework::GradVarName("X")); + auto* out_g = context.Input(framework::GradVarName("Out")); auto* filter_g = context.Output(framework::GradVarName("Filter")); auto* padding_data_g = context.Output(framework::GradVarName("PaddingData")); auto* in = context.Input("X"); auto* filter = context.Input("Filter"); - int context_start = context.Attr("context_start"); - int context_length = context.Attr("context_length"); - int context_stride = context.Attr("context_stride"); - bool padding_trainable = context.Attr("padding_trainable"); + int context_start = context.Attr("contextStart"); + int context_length = context.Attr("contextLength"); + int context_stride = context.Attr("contextStride"); + bool padding_trainable = context.Attr("paddingTrainable"); PADDLE_ENFORCE_EQ(in->lod().size(), 1UL, "Only support one level sequence now."); @@ -115,17 +109,18 @@ class SequenceConvGradKernel : public framework::OpKernel { math::matmul(context.device_context(), *out_g, false, *filter, true, T(1.0), &col, T(1.0)); } - paddle::operators::math::ContextProjectFunctor - seq_project_functor; + math::ContextProjectFunctor seq_project_functor; + math::ContextProjectGradFunctor seq_project_grad_functor; if (in_g) { in_g->mutable_data(context.GetPlace()); in_g->set_lod(in->lod()); set_zero(context.device_context(), in_g, static_cast(0)); - seq_project_functor(context.device_context(), *in_g, *padding_data_g, col, - padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, true, true, false); + seq_project_grad_functor(context.device_context(), *in_g, *padding_data_g, + col, padding_trainable, context_start, + context_length, context_stride, up_pad, down_pad, + true, false); } if (padding_trainable && padding_data_g) { @@ -133,9 +128,10 @@ class SequenceConvGradKernel : public framework::OpKernel { set_zero(context.device_context(), padding_data_g, static_cast(0)); LoDTensor* input = const_cast(in); - seq_project_functor(context.device_context(), *input, *padding_data_g, - col, padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, true, false, true); + seq_project_grad_functor(context.device_context(), *input, + *padding_data_g, col, padding_trainable, + context_start, context_length, context_stride, + up_pad, down_pad, false, true); } if (filter_g) { @@ -150,15 +146,9 @@ class SequenceConvGradKernel : public framework::OpKernel { padding_data = context.Input("PaddingData"); } - sequence_width = static_cast(in->dims()[1]); - - LoDTensor* input = const_cast(in); - Tensor* pad_data = const_cast(padding_data); - - seq_project_functor(context.device_context(), *input, *pad_data, col, + seq_project_functor(context.device_context(), *in, *padding_data, col, padding_trainable, context_start, context_length, - context_stride, up_pad, down_pad, false, false, - false); + context_stride, up_pad, down_pad); math::matmul(context.device_context(), col, true, out_grad, false, T(1.0), &filter_grad, T(1.0)); diff --git a/paddle/operators/sequence_pool_op.cc b/paddle/operators/sequence_pool_op.cc index 6d600c27271c660f0cf933e8bd05455df61740ec..63050a4ec2f16800ae6f8ee077fd0c1daa8eff87 100644 --- a/paddle/operators/sequence_pool_op.cc +++ b/paddle/operators/sequence_pool_op.cc @@ -39,39 +39,42 @@ class SequencePoolOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor), output of SequencePoolOp, which does not contain LoD " "infomation."); - AddAttr( - "strategy", - "(int, default AVERAGE) the pooling strategy of SequencePoolOp.") - .SetDefault(AVERAGE) - .InEnum({AVERAGE, SUM, SQRT, MAX, LAST, FIRST}); + AddAttr( + "pooltype", + "(int, default AVERAGE) the pooling pooltype of SequencePoolOp.") + .SetDefault("AVERAGE") + .InEnum({"AVERAGE", "SUM", "SQRT", "LAST", "FIRST", "MAX"}); AddComment(R"DOC( - SequencePoolOp pools features of all time-steps of each instance. - - It supports six pooling strategy: - - AVERAGE: Out[i] = average_{for each instance in i-th sequence}{X[i]} - - SUM: Out[i] = sum_{for each instance in i-th sequence}{X[i]} - - SQRT: Out[i] = sum_{for each instance in i-th sequence}{X[i]} - / sqrt(i-th sequence length) - - LAST: Out[i] = last instance in i-th sequence X[i] - - FIRST: Out[i] = first instance in i-th sequence X[i] - - MAX: Out[i] = max_{for each instance in i-th sequence}{X[i]} - - For a mini-batch of 3 variable-length sentences, containing 2, 3, and 2 time-steps: - - Assume X is a [7,M,N] LoDTensor, and X->lod()[0] = [0, 2, 5, 7], 7=2+3+2. - Besides, for the sake of simplicity, we assume M=1 and N=1, - and the value of X = [[1, 3], [2, 4, 6], [5, 1]]. - - Thus, Out is a [3,1,1] Tensor without LoD infomation. - And for different strategy, the value of Out is as follows: - - - AVERAGE: [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 - - SUM: [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 - - SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), +Sequence Pool Operator. + +The SequencePoolOp pools features of all time-steps of each instance. +It supports six pooling types: +1. AVERAGE: Out[i] = $$avg(X_i)$$ +2. SUM: Out[i] = $$\sum_jX_{ij}$$ +3. SQRT: Out[i] = $$\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}$$ +4. LAST: Out[i] = last instance in i-th sequence X[i] +5. FIRST: Out[i] = first instance in i-th sequence X[i] +6. MAX: Out[i] = $$max(X_i)$$ + +The following example explains how this works: +For a mini-batch of 3 variable-length sentences, +containing 2, 3, and 2 time-steps: + +Assume X is a [7,M,N] LoDTensor, and X->lod()[0] = [0, 2, 5, 7], 7=2+3+2. +Besides, for the sake of simplicity, we assume M=1 and N=1, +and the value of X = [[1, 3], [2, 4, 6], [5, 1]]. + +Thus, Out is a [3,1,1] Tensor without LoD infomation. +And for different pooltype, the value of Out is as follows: + +- AVERAGE: [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 +- SUM: [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 +- SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), 6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2) - - MAX: [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) - - LAST: [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) - - FIRST: [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) +- MAX: [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) +- LAST: [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) +- FIRST: [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) + )DOC"); } }; diff --git a/paddle/operators/sequence_pool_op.h b/paddle/operators/sequence_pool_op.h index 07bf61df45bf51c8648180ffc9eb97306865fab6..e0e0493fe0ef7e1963ce5c2e3f37c164a605809b 100644 --- a/paddle/operators/sequence_pool_op.h +++ b/paddle/operators/sequence_pool_op.h @@ -29,22 +29,13 @@ template using EigenMatrix = framework::EigenMatrix; -enum SeqPoolType { - AVERAGE = 0, - SUM = 1, - SQRT = 2, // square_root_n - MAX = 3, - LAST = 4, - FIRST = 5 -}; - template class SequencePoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* in = context.Input("X"); auto* out = context.Output("Out"); - int strategy = context.Attr("strategy"); + std::string pooltype = context.Attr("pooltype"); auto dims = in->dims(); auto lod = in->lod(); @@ -71,28 +62,21 @@ class SequencePoolKernel : public framework::OpKernel { auto in_e = EigenMatrix::From(in_t, framework::make_ddim({h, w})); auto out_e = EigenVector::Flatten(out_t); - switch (strategy) { - case AVERAGE: - out_e.device(place) = in_e.mean(Eigen::array({{0}})); - break; - case SUM: - out_e.device(place) = in_e.sum(Eigen::array({{0}})); - break; - case SQRT: - out_e.device(place) = in_e.sum(Eigen::array({{0}})) / - std::sqrt(static_cast(h)); - break; - case MAX: - out_e.device(place) = in_e.maximum(Eigen::array({{0}})); - break; - case LAST: - out_e.device(place) = in_e.chip(h - 1, 0); - break; - case FIRST: - out_e.device(place) = in_e.chip(0, 0); - break; - default: - PADDLE_THROW("unsupported pooling strategy"); + if (pooltype == "AVERAGE") { + out_e.device(place) = in_e.mean(Eigen::array({{0}})); + } else if (pooltype == "SUM") { + out_e.device(place) = in_e.sum(Eigen::array({{0}})); + } else if (pooltype == "SQRT") { + out_e.device(place) = in_e.sum(Eigen::array({{0}})) / + std::sqrt(static_cast(h)); + } else if (pooltype == "MAX") { + out_e.device(place) = in_e.maximum(Eigen::array({{0}})); + } else if (pooltype == "LAST") { + out_e.device(place) = in_e.chip(h - 1, 0); + } else if (pooltype == "FIRST") { + out_e.device(place) = in_e.chip(0, 0); + } else { + PADDLE_THROW("unsupported pooling pooltype"); } } } @@ -105,15 +89,15 @@ class SequencePoolGradKernel : public framework::OpKernel { auto* in = context.Input("X"); auto* in_g = context.Output(framework::GradVarName("X")); auto* out_g = context.Input(framework::GradVarName("Out")); - int strategy = context.Attr("strategy"); + std::string pooltype = context.Attr("pooltype"); auto dims = in->dims(); auto lod = in->lod()[0]; int64_t w = in->numel() / dims[0]; in_g->mutable_data(context.GetPlace()); - if (strategy == LAST || strategy == FIRST) { - // set X@Grad be zero at first when strategy is LAST/FIRST + if (pooltype == "LAST" || pooltype == "FIRST") { + // set X@Grad be zero at first when pooltype is LAST/FIRST math::SetConstant functor; functor(context.device_context(), in_g, 0); } @@ -127,41 +111,33 @@ class SequencePoolGradKernel : public framework::OpKernel { auto out_g_e = EigenMatrix::From(out_g_t, {1, w}); Eigen::DSizes bcast(h, 1); - switch (strategy) { - case AVERAGE: - in_g_e.device(place) = (out_g_e / static_cast(h)).broadcast(bcast); - break; - case SUM: - in_g_e.device(place) = (out_g_e).broadcast(bcast); - break; - case SQRT: - in_g_e.device(place) = - (out_g_e / std::sqrt(static_cast(h))).broadcast(bcast); - break; - case MAX: { - auto in_t = - in->Slice(static_cast(lod[i]), static_cast(lod[i + 1])); - Eigen::Map> - in_t_map(in_t.data(), h, w); - int row_id; - Eigen::array extents{{1, 1}}; - for (int col_id = 0; col_id < w; col_id++) { - in_t_map.col(col_id).maxCoeff(&row_id); - Eigen::array in_offsets{{row_id, col_id}}; - Eigen::array out_offsets{{0, col_id}}; - in_g_e.slice(in_offsets, extents).device(place) = - out_g_e.slice(out_offsets, extents); - } - break; + if (pooltype == "AVERAGE") { + in_g_e.device(place) = (out_g_e / static_cast(h)).broadcast(bcast); + } else if (pooltype == "SUM") { + in_g_e.device(place) = (out_g_e).broadcast(bcast); + } else if (pooltype == "SQRT") { + in_g_e.device(place) = + (out_g_e / std::sqrt(static_cast(h))).broadcast(bcast); + } else if (pooltype == "MAX") { + auto in_t = + in->Slice(static_cast(lod[i]), static_cast(lod[i + 1])); + Eigen::Map> + in_t_map(in_t.data(), h, w); + int row_id; + Eigen::array extents{{1, 1}}; + for (int col_id = 0; col_id < w; col_id++) { + in_t_map.col(col_id).maxCoeff(&row_id); + Eigen::array in_offsets{{row_id, col_id}}; + Eigen::array out_offsets{{0, col_id}}; + in_g_e.slice(in_offsets, extents).device(place) = + out_g_e.slice(out_offsets, extents); } - case LAST: - in_g_e.chip(h - 1, 0).device(place) = out_g_e; - break; - case FIRST: - in_g_e.chip(0, 0).device(place) = out_g_e; - break; - default: - PADDLE_THROW("unsupported pooling strategy"); + } else if (pooltype == "LAST") { + in_g_e.chip(h - 1, 0).device(place) = out_g_e; + } else if (pooltype == "FIRST") { + in_g_e.chip(0, 0).device(place) = out_g_e; + } else { + PADDLE_THROW("unsupported pooling pooltype"); } } } diff --git a/paddle/operators/sequence_softmax_op.cc b/paddle/operators/sequence_softmax_op.cc index c891ab1fdcbb167453462c45b00b4632e663dd0e..32c15025660ebf0baf317e269a33c047e6844219 100644 --- a/paddle/operators/sequence_softmax_op.cc +++ b/paddle/operators/sequence_softmax_op.cc @@ -43,20 +43,24 @@ class SequenceSoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { "(LoDTensor) 1-D or 2-D output LoDTensor with the 2-nd dimension " "of length 1."); AddComment(R"DOC( -SequenceSoftmaxOp computes softmax activation among all time-steps for each +Sequence Softmax Operator. + +SequenceSoftmaxOp computes the softmax activation among all time-steps for each sequence. The dimension of each time-step should be 1. Thus, the shape of -input Tensor can be either [N, 1] or [N], where N is the sum of all sequences' -lengths. +input Tensor can be either [N, 1] or [N], where N is the sum of the length +of all sequences. -Equation: +The algorithm works as follows: for i-th sequence in a mini-batch: - Out(X[lod[i]:lod[i+1]], :) = - exp(X[lod[i]:lod[i+1], :]) / sum(exp(X[lod[i]:lod[i+1], :])) + $$Out(X[lod[i]:lod[i+1]], :) = + \frac{\exp(X[lod[i]:lod[i+1], :])} + {\sum(\exp(X[lod[i]:lod[i+1], :]))}$$ For example, for a mini-batch of 3 sequences with variable-length, each containing 2, 3, 2 time-steps, the lod of which is [0, 2, 5, 7], then softmax will be computed among X[0:2, :], X[2:5, :], X[5:7, :] and N turns out to be 7. + )DOC"); } }; diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 939176c73dc21dc662b1aaf23d8077c6856a5650..72f4e4d5cbcd692423fa2a3e9ec8e7033b552c3c 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -45,15 +45,17 @@ class SGDOpMaker : public framework::OpProtoAndCheckerMaker { public: SGDOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Param", "Input parameter"); - AddInput("LearningRate", "Learning rate of SGD"); - AddInput("Grad", "Input gradient"); - AddOutput("ParamOut", "output parameter"); + AddInput("Param", "(Tensor) Input parameter"); + AddInput("LearningRate", "(Tensor) Learning rate of SGD"); + AddInput("Grad", "(Tensor) Input gradient"); + AddOutput("ParamOut", "(Tensor) Output parameter"); AddComment(R"DOC( -Simplest sgd algorithm. +SGD operator -param_out = param - learning_rate * grad; +This operator implements one step of the stochastic gradient descent algorithm. + +$$param_out = param - learning_rate * grad$$ )DOC"); } diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc index e781c8db208464cb94d94d1914e50f5aba3db2c6..d9e40546523c60b0a7eec2e0593446258996ba58 100644 --- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc +++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc @@ -107,26 +107,28 @@ class SigmoidCrossEntropyWithLogitsOpMaker AddComment(R"DOC( SigmoidCrossEntropyWithLogits Operator. -This measures the elementwise probability error in discrete classification tasks +This measures the element-wise probability error in classification tasks in which each class is independent. This can be thought of as predicting labels -for a data-point that are not mutually exclusive. For example, a news article -can be about politics, technology or sports at the same time or none of these. +for a data-point, where labels are not mutually exclusive. +For example, a news article can be about politics, technology or sports +at the same time or none of these. The logistic loss is given as follows: - loss = -Labels * log(sigmoid(X)) - (1 - Labels) * log(1 - sigmoid(X)) + $$loss = -Labels * \log(\sigma(X)) - (1 - Labels) * \log(1 - \sigma(X))$$ -We know that sigmoid(X) = (1 / (1 + exp(-X))). By substituting this we get +We know that $$\sigma(X) = (1 / (1 + \exp(-X)))$$. By substituting this we get: - loss = X - X * Labels + log(1 + exp(-X)) + $$loss = X - X * Labels + \log(1 + \exp(-X))$$ -For stability and to prevent overflow of exp(-X) when X < 0, -we can reformulate the loss as follows: +For stability and to prevent overflow of $$\exp(-X)$$ when X < 0, +we reformulate the loss as follows: - loss = max(X, 0) - X * Labels + log(1 + exp(-abs(X))) + $$loss = \max(X, 0) - X * Labels + \log(1 + \exp(-|X|))$$ Both the input `X` and `Labels` can carry the LoD (Level of Details) information. However the output only shares the LoD with input `X`. + )DOC"); } }; diff --git a/paddle/operators/sign_op.cc b/paddle/operators/sign_op.cc index 1b2f879d6d305e4e77be41683d8249904337a6f8..08bf2e4e7cc101a3bcc907d3b40ee82347b39f80 100644 --- a/paddle/operators/sign_op.cc +++ b/paddle/operators/sign_op.cc @@ -38,9 +38,10 @@ class SignOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) Input tensor of sign operator."); AddOutput("Out", "(Tensor) Output tensor of sign operator."); - AddComment(R"DOC(Sign operator + AddComment(R"DOC( +Sign operator -The equation is: Out = X.sign() +$$Out = X.sign()$$ )DOC"); } }; diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index 758481943d463f22eb6c6e0be9a99ad99161da5b..ebf7b43700a7498aa18b5f648b0b8c2c4e7b442b 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -77,14 +77,17 @@ class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { "A float scalar with default value 3.0.") .SetDefault(3.0); AddComment(R"DOC( -Compute smooth l1 loss for input and target. The operator take the 1st -dimension of input as batch size. For each instance, it will compute -smooth l1 loss element by element first and sum all losses to one value. -So the output shape is [batch_size, 1]. +Smooth L1 Loss Operator. + +This operator computes the smooth l1 loss for input and target. +The operator takes the first dimension of input as the batch size. +For each instance, it computes the smooth l1 loss element by element first +and then sums all the losses. So the resulting output shape +is [batch_size, 1]. The equation is: -loss = 0.5 * (sigma * (x-y))^2 if abs(x - y) < 1 / sigma^2 - abs(x - y) - 0.5 / sigma^2 otherwise +loss = $$0.5 * (\sigma * (x-y))^2$$ if $$|x - y| < 1 /({\sigma}^2)$$ + $$\frac{|x - y| - 0.5}{{\sigma}^2}$$ otherwise )DOC"); } diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index 00fd0b32a9b3c0dd9fedf7b7621b1f15e5c4ce93..93f89e33a73c5f4c6c0e5a8793a0abe7c692b656 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -44,20 +44,23 @@ class SoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { "2-D with shape [batch_size, input_feature_dimensions]."); AddOutput("Y", "The normalized values with the same shape as X."); AddComment(R"DOC( -The input of softmax operator is a 2-D tensor with shape N x K (N is the +Softmax Operator. + +The input of the softmax operator is a 2-D tensor with shape N x K (N is the batch_size, K is the dimension of input feature). The output tensor has the same shape as the input tensor. For each row of the input tensor, the softmax operator squashes the K-dimensional vector of arbitrary real values to a K-dimensional vector of real -values in the range [0, 1] that add up to 1. Specifically, it computes the -exponential of the given dimension and the sum of exponential values of all -the other dimensions in the K-dimensional vector input. Then the ratio of the -exponential of the given dimension and the sum of exponential values of all -the other dimensions is the output of the softmax operator. +values in the range [0, 1] that add up to 1. +It computes the exponential of the given dimension and the sum of exponential +values of all the other dimensions in the K-dimensional vector input. +Then the ratio of the exponential of the given dimension and the sum of +exponential values of all the other dimensions is the output of the softmax +operator. For each row `i` and each column `j` in input X, we have: - Y[i, j] = exp(X[i, j]) / sum_j(exp(X[i, j])) + $$Y[i, j] = \frac{\exp(X[i, j])}{\sum_j(exp(X[i, j])}$$ )DOC"); } diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index 942fbb42df8bb90b86bd097832a15b320a857750..a006e0a595057d019b443d22cea4bdf171a6ee0b 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -32,9 +32,9 @@ class SoftmaxWithCrossEntropyOpMaker AddInput("Label", "(Tensor, default: Tensor), The ground truth which is a 2-D " "tensor. " - "If softLable is set to 0, Label is a Tensor with shape [N x " - "1]. " - "If softLable is set to 1, Label is a Tensor " + "If softLabel is set to false, Label is a Tensor with shape " + "[N x 1]." + "If softLabel is set to true, Label is a Tensor " "with shape [N x K]."); AddOutput( "Softmax", @@ -51,28 +51,34 @@ class SoftmaxWithCrossEntropyOpMaker "the given labels as soft labels.") .SetDefault(false); AddComment(R"DOC( -Cross entropy loss with softmax are used as the output layer extensively. This +Softmax With Cross Entropy Operator. + +Cross entropy loss with softmax is used as the output layer extensively. This operator computes the softmax normalized values for each row of the input -tensor, after which cross-entropy loss is then computed. This provides a more +tensor, after which cross-entropy loss is computed. This provides a more numerically stable gradient. -Because this operators performs a softmax on logits internally, it expects -unscaled logits. Please do not call this op with the output of softmax operator, -which will produce incorrect results. +Because this operator performs a softmax on logits internally, it expects +unscaled logits. This operator should not be used with the output of +softmax operator since that would produce incorrect results. -This operators expects mutually exclusive hard labels, each sample in a batch -is in exactly one class with probabilities 1. Each sample in the batch with one -and only one label. +When the attribute softLabel is set false, this operators expects mutually +exclusive hard labels, each sample in a batch is in exactly one class with a +probability of 1.0. Each sample in the batch will have a single label. -Equation: +The equation is as follows: -1) hard label (one-hot label) +1) Hard label (one-hot label, so every sample has exactly one class) -Loss_j = -\text{Logit}_{Label_j} + \log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right), j = 1, ..., K +$$Loss_j = \f$ -\text{Logit}_{Label_j} + +\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right), +j = 1, ..., K $\f$$ -2) soft label (a distribution over all classes) +2) Soft label (each sample can have a distribution over all classes) -Loss_j = -\sum_{i=0}^{K}\text{Label}_i\left(\text{Logit}_i-\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right)\right), j = 1,...,K +$$Loss_j = \f$ -\sum_{i=0}^{K}\text{Label}_i\left(\text{Logit}_i - +\log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right)\right), +j = 1,...,K $\f$$ )DOC"); } diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc index 1ef314b77f0fdd395ddb0cecf8f29e97559cb7ca..275b25e96aa75fdbcb7275e272c49ea8d278d2c8 100644 --- a/paddle/operators/split_op.cc +++ b/paddle/operators/split_op.cc @@ -67,30 +67,38 @@ class SplitOpMaker : public framework::OpProtoAndCheckerMaker { public: SplitOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensor of split operator."); - AddOutput("Out", "the output tensors of split operator.").AsDuplicable(); + AddInput("X", "(Tensor) Input tensor of the split operator."); + AddOutput("Out", "(Tensor) Output tensors of the split operator.") + .AsDuplicable(); AddComment(R"DOC( - Split the input tensor into multiple sub-tensors. - Example: - Input = [[1,2], - [3,4], - [5,6]] - sections = [2,1] - axis = 0 - Output[0] = [[1,2], - [3,4]] - Output[1] = [[5,6]] +Split operator + +This operator splits the input tensor into multiple sub-tensors. + +Example: + Input = [[1,2], + [3,4], + [5,6]] + sections = [2,1] + axis = 0 + Output[0] = [[1,2], + [3,4]] + Output[1] = [[5,6]] )DOC"); AddAttr>("sections", - "the length for each" - "output along with the specify axis.") + "(vector) " + "the length of each output along the " + "specified axis.") .SetDefault(std::vector{}); AddAttr("num", - "number of the sub-tensors, it must evenly divide " + "(int, default 0)" + "Number of sub-tensors. This must evenly divide " "Input.dims()[axis]") .SetDefault(0); - AddAttr("axis", "The axis which the input will be splited on.") + AddAttr("axis", + "(int, default 0) " + "The axis which the input will be splited on.") .SetDefault(0); } }; diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc index e360c19b47eae7fc32ae66f9e4e3873bff211b04..bec2a2c18ae8da892ee7d71f45afe53c887c0f57 100644 --- a/paddle/operators/squared_l2_distance_op.cc +++ b/paddle/operators/squared_l2_distance_op.cc @@ -59,23 +59,26 @@ class SquaredL2DistanceOpMaker : public framework::OpProtoAndCheckerMaker { SquaredL2DistanceOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "Input of SquaredL2DistanceOp."); - AddInput("Y", "Target of SquaredL2DistanceOp."); + AddInput("X", "(Tensor) Input of SquaredL2DistanceOp."); + AddInput("Y", "(Tensor) Target of SquaredL2DistanceOp."); AddOutput("sub_result", - "Buffering substraction result which " + "(Tensor) Buffering subtraction result which " "will be reused in backward.") .AsIntermediate(); - AddOutput("Out", "Squared l2 distance between input and target."); + AddOutput("Out", "(Tensor) Squared l2 distance between input and target."); AddComment(R"DOC( - SquaredL2DistanceOp will cacluate the squared L2 distance for - input and target. Number of distance value equals to the - first dimension of input. First dimension of target could be equal to - input or to 1. If the first dimension of target is 1, SquaredL2DistanceOp - will broadcast target's first dimension to input's first dimension. - You can decide whether calculate the gradient of input and target. - - Both the input X and Y can carry the LoD (Level of Details) information, - or not. But the output only shares the LoD with input X. +SquaredL2Distance operator + +This operator will cacluate the squared L2 distance for the input and +the target. Number of distance value will be equal to the first dimension +of input. First dimension of the target could be equal to the input or to 1. +If the first dimension of target is 1, the operator will broadcast target's +first dimension to input's first dimension. During backward propagation, +the user can decide whether to calculate the gradient of the input or +the target or both. + +Both the input X and Y can carry the LoD (Level of Details) information. +However, the output only shares the LoD information with input X. )DOC"); } }; diff --git a/paddle/operators/squared_l2_norm_op.cc b/paddle/operators/squared_l2_norm_op.cc index 42ad87e65a85355e1b9b927dcef9ebbb88cde717..3c10e6159f44bc8c21b1e79aefaa962c7a2b64ed 100644 --- a/paddle/operators/squared_l2_norm_op.cc +++ b/paddle/operators/squared_l2_norm_op.cc @@ -52,13 +52,13 @@ class SquaredL2NormOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) The input of squared_l2_norm op."); - AddOutput("Out", "(Float) The output of squared_l2_norm op."); + AddOutput("Out", "(Scalar) The output of squared_l2_norm op."); AddComment(R"DOC( SquaredL2Norm Operator. Computes the squared L2 norm of a tensor. -Out = sum (X ** 2) +$$Out = \sum_{i} X_{i}^2$$ )DOC"); } diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index ca36ad764c8a4cb5f6c58d3ac3d9ff4a588f3200..d9d3dd6e37a8ffd7aa7a2e6f47a1c225474f630b 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -45,13 +45,15 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: SumOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of sum operator.").AsDuplicable(); - AddOutput("Out", "the output tensor of sum operator."); + AddInput("X", "(vector) The input tensors of sum operator.") + .AsDuplicable(); + AddOutput("Out", "(Tensor) The output tensor of sum operator."); AddComment(R"DOC( -Sum the input tensors. +Sum operator. -All the inputs can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with the first input. +This operators sums the input tensors. All the inputs can carry the +LoD (Level of Details) information. However, the output only shares +the LoD information with the first input. )DOC"); } }; diff --git a/paddle/operators/sum_op.h b/paddle/operators/sum_op.h index f2f2c67bc395ea245798b537144dd88a816f4a85..ad441a598040aca71a72c9c03d477934d14e9a8b 100644 --- a/paddle/operators/sum_op.h +++ b/paddle/operators/sum_op.h @@ -29,22 +29,27 @@ template class SumKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto& in_vars = context.MultiInputVar("X"); + auto in_vars = context.MultiInputVar("X"); int N = in_vars.size(); auto out_var = context.OutputVar("Out"); + bool in_place = out_var == in_vars[0]; + if (out_var->IsType()) { auto* out = context.Output("Out"); out->mutable_data(context.GetPlace()); auto result = EigenVector::Flatten(*out); - math::SetConstant constant_functor; - constant_functor(context.device_context(), out, 0.0); + if (!in_place) { + math::SetConstant constant_functor; + constant_functor(context.device_context(), out, 0.0); + } math::SelectedRowsAddToTensor functor; auto place = context.GetEigenDevice(); - for (int i = 0; i < N; i++) { + // If in_place, just skip the first tensor + for (int i = in_place ? 1 : 0; i < N; i++) { if (in_vars[i]->IsType()) { auto& in_t = in_vars[i]->Get(); auto in = EigenVector::Flatten(in_t); @@ -57,6 +62,7 @@ class SumKernel : public framework::OpKernel { } } } else if (out_var->IsType()) { + PADDLE_ENFORCE(!in_place, "SelectedRows not support inplace sum now"); auto* out = context.Output("Out"); auto* out_value = out->mutable_value(); diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc index d5c2c91a5fb0f639ea84d13e27de8271218da54f..16ae925eb5cab1c05f3bc376972cabadc4367d20 100644 --- a/paddle/operators/top_k_op.cc +++ b/paddle/operators/top_k_op.cc @@ -48,16 +48,20 @@ class TopkOpMaker : public framework::OpProtoAndCheckerMaker { public: TopkOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The input of Topk op"); - AddOutput("Out", "The output tensor of Topk op"); - AddOutput("Indices", "The indices of Topk elements of input"); - AddComment( - R"DOC(If the input is a vector (1d tensor), finds the k largest entries in the vector and outputs their values and indices as vectors. Thus values[j] is the j-th largest entry in input, and its index is indices[j]. + AddInput("X", "(Tensor) The input of Topk op"); + AddOutput("Out", "(Tensor) The output tensor of Topk op"); + AddOutput("Indices", "(Tensor) The indices of Topk elements of input"); + AddComment(R"DOC( +Top K operator - For matrices, computes the top k entries in each row. )DOC"); +If the input is a vector (1d tensor), this operator finds the k largest +entries in the vector and outputs their values and indices as vectors. +Thus values[j] is the j-th largest entry in input, and its index is indices[j]. + +For matrices, this operator computes the top k entries in each row. )DOC"); AddAttr("k", - "Number of top elements to look for along the last " - "dimension (along each row for matrices).") + "(int, default 1) Number of top elements to look for along " + "the last dimension (along each row for matrices).") .SetDefault(1); } }; @@ -66,6 +70,7 @@ class TopkOpMaker : public framework::OpProtoAndCheckerMaker { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(top_k, ops::TopkOp, ops::TopkOpMaker); +REGISTER_OPERATOR(top_k, ops::TopkOp, ops::TopkOpMaker, + paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL(top_k, ops::TopkKernel); diff --git a/paddle/operators/top_k_op.cu b/paddle/operators/top_k_op.cu index 7be6932f1e301d06e0e232367a38bfa673ff45be..7851c71bbe9fe73402968ce14f6db0df523cd6d3 100644 --- a/paddle/operators/top_k_op.cu +++ b/paddle/operators/top_k_op.cu @@ -23,9 +23,9 @@ using Tensor = framework::Tensor; template struct Pair { __device__ __forceinline__ Pair() {} - __device__ __forceinline__ Pair(T value, int id) : v(value), id(id) {} + __device__ __forceinline__ Pair(T value, int64_t id) : v(value), id(id) {} - __device__ __forceinline__ void set(T value, int id) { + __device__ __forceinline__ void set(T value, int64_t id) { v = value; id = id; } @@ -48,7 +48,7 @@ struct Pair { } T v; - int id; + int64_t id; }; template @@ -197,7 +197,7 @@ __device__ __forceinline__ void ThreadGetTopK(Pair topk[], int& beam, template __device__ __forceinline__ void BlockReduce(Pair* sh_topk, int* maxid, Pair topk[], T** topVal, - int** topIds, int& beam, int& k, + int64_t** topIds, int& beam, int& k, const int tid, const int warp) { while (true) { __syncthreads(); @@ -249,7 +249,7 @@ __device__ __forceinline__ void BlockReduce(Pair* sh_topk, int* maxid, * 4. go to the first setp, until get the topk value. */ template -__global__ void KeMatrixTopK(T* output, int output_stride, int* indices, +__global__ void KeMatrixTopK(T* output, int output_stride, int64_t* indices, const T* src, int lds, int dim, int k) { __shared__ Pair sh_topk[BlockSize]; __shared__ int maxid[BlockSize / 2]; @@ -293,7 +293,7 @@ class TopkOpCUDAKernel : public framework::OpKernel { T* output_data = output->mutable_data(ctx.GetPlace()); // FIXME(typhoonzero): data is always converted to type T? - int* indices_data = indices->mutable_data(ctx.GetPlace()); + int64_t* indices_data = indices->mutable_data(ctx.GetPlace()); size_t input_height = input->dims()[0]; size_t input_width = input->dims()[1]; diff --git a/paddle/operators/top_k_op.h b/paddle/operators/top_k_op.h index 4b248faa120bcfd20e70d288cce2d485d3e6371e..bc8563717a21bd5b3d8fc87f689657990066957b 100644 --- a/paddle/operators/top_k_op.h +++ b/paddle/operators/top_k_op.h @@ -40,7 +40,7 @@ class TopkKernel : public framework::OpKernel { const size_t k = static_cast(ctx.Attr("k")); T* output_data = output->mutable_data(ctx.GetPlace()); - T* indices_data = indices->mutable_data(ctx.GetPlace()); + int64_t* indices_data = indices->mutable_data(ctx.GetPlace()); auto eg_input = EigenMatrix::From(*input); @@ -66,7 +66,7 @@ class TopkKernel : public framework::OpKernel { }); for (size_t j = 0; j < k; j++) { output_data[i * k + j] = vec[j].first; - indices_data[i * k + j] = vec[j].second; + indices_data[i * k + j] = int64_t(vec[j].second); } } } diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc index d785e57c830439ad80005d9a3d4bb77faf1ae1b9..94de3d5069017a7ca818e246ad574c4db92d8006 100644 --- a/paddle/operators/transpose_op.cc +++ b/paddle/operators/transpose_op.cc @@ -32,7 +32,7 @@ class TransposeOp : public framework::OperatorWithKernel { size_t axis_size = axis.size(); PADDLE_ENFORCE_EQ(x_rank, axis_size, - "the input tensor's rank(%d) " + "The input tensor's rank(%d) " "should be equal to the axis's size(%d)", x_rank, axis_size); @@ -64,12 +64,14 @@ class TransposeOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor)The output tensor"); AddAttr>( "axis", - "(vector)a list of values, and the size of the list should be " + "(vector)A list of values, and the size of the list should be " "the same with the input tensor rank, the tensor will " "permute the axes according the the values given"); AddComment(R"DOC( -The Tensor will be permuted according to the axis values given. -The op is very much like the numpy.transpose function in python +Transpose Operator. + +The input tensor will be permuted according to the axis values given. +The op functions similar to how numpy.transpose works in python. For example: >> input = numpy.arange(6).reshape((2,3)) >> input @@ -83,6 +85,7 @@ For example: [2, 5]]) So, given a input tensor of shape(N, C, H, W) and the axis is {0, 2, 3, 1}, the output tensor shape will be (N, H, W, C) + )DOC"); } }; diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index 82f9b8fbf1094bde1def83b9a1c464207b7e4669..cd22c561acb6cbd66faf1d3d6ee21779001a2795 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -74,18 +74,30 @@ class UniformRandomOpMaker : public framework::OpProtoAndCheckerMaker { UniformRandomOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "The output tensor of uniform random op"); - AddComment(R"DOC(Uniform random operator. -Used to initialize tensor with uniform random generator. + AddOutput("Out", "(Tensor) The output tensor of uniform random op"); + AddComment(R"DOC( +Uniform random operator. + +This operator initializes a tensor with random values sampled from a +uniform distribution. + )DOC"); - AddAttr>("shape", "the dimension of random tensor"); - AddAttr("min", "Minimum value of uniform random").SetDefault(-1.0f); - AddAttr("max", "Maximun value of uniform random").SetDefault(1.0f); + AddAttr>("shape", + "(vector) The shape of the output tensor"); + AddAttr("min", + "(float, default -1.0) " + "Minimum value of uniform random") + .SetDefault(-1.0f); + AddAttr("max", + "(float, default 1.0) " + "Maximun value of uniform random") + .SetDefault(1.0f); AddAttr("seed", - "Random seed of uniform random. " - "0 means generate a seed by system") + "(int, default 0) " + "Random seed used for generating samples. " + "0 means use a seed generated by the system.") .SetDefault(0); - AddAttr("data_type", "output tensor data type") + AddAttr("data_type", "(int, default 5(FP32)) Output tensor data type") .SetDefault(framework::DataType::FP32); } }; diff --git a/paddle/optimizer/parameter_optimizer_test.cpp b/paddle/optimizer/parameter_optimizer_test.cpp index c88fa11748716693355042d1784b33d7cfb616f1..c99b2254ac6974343206c237377b2440ba8efdf8 100644 --- a/paddle/optimizer/parameter_optimizer_test.cpp +++ b/paddle/optimizer/parameter_optimizer_test.cpp @@ -85,7 +85,7 @@ public: for (size_t i = 0; i < opts_.size(); ++i) { int s = 0; float* newp = (float*)opts_[i]->get_weight(&s); - EXPECT_EQ(s, kSize); + EXPECT_EQ(static_cast(s), kSize); for (size_t j = 0; j < kSize; ++j) { EXPECT_EQ(newp[j], (*p)[j]); } diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt index eb850b658583f2256629d63fdb64248dbf249937..bd86a9fe268c277065cd450f91b544def6c4d32f 100644 --- a/paddle/platform/CMakeLists.txt +++ b/paddle/platform/CMakeLists.txt @@ -9,7 +9,6 @@ cc_test(place_test SRCS place_test.cc DEPS place glog gflags) add_subdirectory(dynload) cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece) -cc_test(environment_test SRCS environment_test.cc DEPS stringpiece) IF(WITH_GPU) set(GPU_CTX_DEPS dynload_cuda dynamic_loader) diff --git a/paddle/platform/environment.h b/paddle/platform/environment.h deleted file mode 100644 index 4edcce932edc61453cef74f2c4ee0f72496b3677..0000000000000000000000000000000000000000 --- a/paddle/platform/environment.h +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include - -#include "paddle/platform/enforce.h" -#include "paddle/string/piece.h" - -extern char** environ; // for environment variables - -namespace paddle { -namespace platform { - -inline void SetEnvVariable(const std::string& name, const std::string& value) { - PADDLE_ENFORCE_NE(setenv(name.c_str(), value.c_str(), 1), -1, - "Failed to set environment variable %s=%s", name, value); -} - -inline void UnsetEnvVariable(const std::string& name) { - PADDLE_ENFORCE_NE(unsetenv(name.c_str()), -1, - "Failed to unset environment variable %s", name); -} - -inline bool IsEnvVarDefined(const std::string& name) { - return std::getenv(name.c_str()) != nullptr; -} - -inline std::string GetEnvValue(const std::string& name) { - PADDLE_ENFORCE(IsEnvVarDefined(name), - "Tried to access undefined environment variable %s", name); - return std::getenv(name.c_str()); -} - -inline std::vector GetAllEnvVariables() { - std::vector vars; - for (auto var = environ; *var != nullptr; ++var) { - auto tail = string::Index(*var, "="); - auto name = string::SubStr(*var, 0, tail).ToString(); - vars.push_back(name); - } - return vars; -} - -} // namespace platform -} // namespace paddle diff --git a/paddle/platform/environment_test.cc b/paddle/platform/environment_test.cc deleted file mode 100644 index 5f136527215d6a676cfa1a3b08f09dfd3ab24a90..0000000000000000000000000000000000000000 --- a/paddle/platform/environment_test.cc +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/platform/environment.h" - -#include "glog/logging.h" -#include "gtest/gtest.h" - -TEST(ENVIRONMENT, ACCESS) { - namespace platform = paddle::platform; - namespace string = paddle::string; - - platform::SetEnvVariable("PADDLE_USE_ENV", "TRUE"); - - EXPECT_TRUE(platform::IsEnvVarDefined("PADDLE_USE_ENV")); - EXPECT_EQ(platform::GetEnvValue("PADDLE_USE_ENV"), "TRUE"); - - platform::UnsetEnvVariable("PADDLE_USE_ENV"); - EXPECT_FALSE(platform::IsEnvVarDefined("PADDLE_USE_ENV")); - - platform::SetEnvVariable("PADDLE_USE_ENV1", "Hello "); - platform::SetEnvVariable("PADDLE_USE_ENV2", "World, "); - platform::SetEnvVariable("PADDLE_USE_ENV3", "PaddlePaddle!"); - - std::string env_info; - auto vars = platform::GetAllEnvVariables(); - for_each(vars.begin(), vars.end(), [&](const std::string& var) { - env_info += platform::GetEnvValue(var); - }); - - EXPECT_TRUE(string::Contains(env_info, "Hello World, PaddlePaddle!")); - platform::UnsetEnvVariable("PADDLE_USE_ENV1"); - platform::UnsetEnvVariable("PADDLE_USE_ENV2"); - platform::UnsetEnvVariable("PADDLE_USE_ENV3"); - - env_info.clear(); - vars = platform::GetAllEnvVariables(); - for_each(vars.begin(), vars.end(), [&](const std::string& var) { - env_info += platform::GetEnvValue(var); - }); - - EXPECT_FALSE(string::Contains(env_info, "Hello World, PaddlePaddle!")); - EXPECT_FALSE(platform::IsEnvVarDefined("PADDLE_USE_ENV1")); - EXPECT_FALSE(platform::IsEnvVarDefined("PADDLE_USE_ENV2")); - EXPECT_FALSE(platform::IsEnvVarDefined("PADDLE_USE_ENV3")); -} diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc index 0cab5ffc5609bbd6fd08c74329d8370fb95f8102..f3455a8733862c91eaece629b6684d446672336c 100644 --- a/paddle/platform/gpu_info.cc +++ b/paddle/platform/gpu_info.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include "gflags/gflags.h" #include "paddle/platform/enforce.h" -#include "paddle/platform/environment.h" DEFINE_double(fraction_of_gpu_memory_to_use, 0.95, "Default use 95% of GPU memory for PaddlePaddle," @@ -75,13 +74,6 @@ size_t GpuMaxChunkSize() { GpuMemoryUsage(available, total); - if (IsEnvVarDefined(kEnvFractionGpuMemoryToUse)) { - auto val = std::stod(GetEnvValue(kEnvFractionGpuMemoryToUse)); - PADDLE_ENFORCE_GT(val, 0.0); - PADDLE_ENFORCE_LE(val, 1.0); - FLAGS_fraction_of_gpu_memory_to_use = val; - } - // Reserving the rest memory for page tables, etc. size_t reserving = (1 - FLAGS_fraction_of_gpu_memory_to_use) * total; diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 14adfa1f35225ca5bf0c093dcf75d1c21af69676..5462e6c6c7a840f75f8c15195c6d6910f30ef733 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -129,7 +129,8 @@ void BindProgramDesc(py::module &m) { } return retv; }) - .def("block", &ProgramDescBind::Block, py::return_value_policy::reference) + .def("block", &ProgramDescBind::MutableBlock, + py::return_value_policy::reference) .def("num_blocks", &ProgramDescBind::Size) .def("serialize_to_string", [](ProgramDescBind &program_desc) -> py::bytes { @@ -237,7 +238,9 @@ void BindVarDsec(py::module &m) { .value("SELECTED_ROWS", VarDesc::SELECTED_ROWS) .value("FEED_MINIBATCH", VarDesc::FEED_MINIBATCH) .value("FETCH_LIST", VarDesc::FETCH_LIST) - .value("STEP_SCOPES", VarDesc::STEP_SCOPES); + .value("STEP_SCOPES", VarDesc::STEP_SCOPES) + .value("LOD_RANK_TABLE", VarDesc::LOD_RANK_TABLE) + .value("LOD_TENSOR_ARRAY", VarDesc::LOD_TENSOR_ARRAY); } void BindOpDesc(py::module &m) { diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index bf6e12264269c7603484e0acf502adab25645856..0c528174b2b2b3a27869ed0083fc96b8d90e723b 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -14,18 +14,22 @@ limitations under the License. */ #include "paddle/pybind/protobuf.h" +#include // for call_once +#include +#include "gflags/gflags.h" #include "paddle/framework/backward.h" #include "paddle/framework/executor.h" #include "paddle/framework/feed_fetch_method.h" #include "paddle/framework/framework.pb.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/prune.h" #include "paddle/framework/selected_rows.h" #include "paddle/framework/tensor_array.h" #include "paddle/operators/cond_op.h" #include "paddle/operators/dynamic_recurrent_op.h" #include "paddle/operators/net_op.h" -#include "paddle/operators/recurrent_op.h" #include "paddle/platform/enforce.h" #include "paddle/platform/place.h" #include "paddle/pybind/exception.h" @@ -40,9 +44,27 @@ limitations under the License. */ namespace paddle { namespace pybind { -static size_t UniqueIntegerGenerator() { - static std::atomic generator; - return generator.fetch_add(1); +static size_t UniqueIntegerGenerator(const std::string &prefix) { + static std::unordered_map> generators; + return generators[prefix].fetch_add(1); +} + +std::once_flag gflags_init_flag; + +// TODO(qijun) move init gflags to init.cc +void InitGflags(std::vector &argv) { + std::call_once(gflags_init_flag, [&]() { + int argc = argv.size(); + char **arr = new char *[argv.size()]; + std::string line; + for (size_t i = 0; i < argv.size(); i++) { + arr[i] = &argv[i][0]; + line += argv[i]; + line += ' '; + } + google::ParseCommandLineFlags(&argc, &arr, true); + VLOG(1) << "Init commandline: " << line; + }); } bool IsCompileGPU() { @@ -204,11 +226,17 @@ All parameter, weight, gradient are variables in Paddle. return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_lod_rank_table", + [](Variable &self) { return self.GetMutable(); }, + py::return_value_policy::reference) .def("get_selected_rows", [](Variable &self) -> SelectedRows * { return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_lod_tensor_array", + [](Variable &self) { return self.GetMutable(); }, + py::return_value_policy::reference) #ifdef PADDLE_WITH_CUDA .def("get_communicator", [](Variable &self) -> platform::Communicator * { @@ -254,7 +282,7 @@ All parameter, weight, gradient are variables in Paddle. const std::vector> &targets) { ProgramDescBind prog_with_targets(origin); for (const auto &t : targets) { - prog_with_targets.Block(t[0])->Op(t[1])->MarkAsTarget(); + prog_with_targets.MutableBlock(t[0])->Op(t[1])->MarkAsTarget(); } ProgramDesc pruned_desc; Prune(*prog_with_targets.Proto(), &pruned_desc); @@ -314,7 +342,7 @@ All parameter, weight, gradient are variables in Paddle. PADDLE_ENFORCE(desc.IsInitialized(), "User OpDesc is not initialized, reason %s", desc.InitializationErrorString()); - return OpRegistry::CreateOp(desc, nullptr); + return OpRegistry::CreateOp(desc); }) .def("backward", [](const OperatorBase &forwardOp, @@ -407,25 +435,6 @@ All parameter, weight, gradient are variables in Paddle. return self.UnstackShared(source); }); - // recurrent_op - py::class_(m, "RecurrentOp") - .def_static( - "create", - [](py::bytes protobin) -> operators::RecurrentOp * { - OpDesc desc; - PADDLE_ENFORCE(desc.ParsePartialFromString(protobin), - "Cannot parse user input to OpDesc"); - PADDLE_ENFORCE(desc.IsInitialized(), - "User OpDesc is not initialized, reason %s", - desc.InitializationErrorString()); - auto rnn_op = OpRegistry::CreateOp(desc, nullptr); - return static_cast(rnn_op.release()); - }) - .def("set_stepnet", [](operators::RecurrentOp &self, - const operators::NetOp &net) -> void { - self.set_stepnet(net.Clone()); - }); - py::class_(m, "DynamicRecurrentOp") .def_static("create", @@ -436,7 +445,7 @@ All parameter, weight, gradient are variables in Paddle. PADDLE_ENFORCE(desc.IsInitialized(), "User OpDesc is not initialized, reason %s", desc.InitializationErrorString()); - auto rnn_op = OpRegistry::CreateOp(desc, nullptr); + auto rnn_op = OpRegistry::CreateOp(desc); return static_cast( rnn_op.release()); }) @@ -463,7 +472,7 @@ All parameter, weight, gradient are variables in Paddle. PADDLE_ENFORCE(desc.IsInitialized(), "User OpDesc is not initialized, reason %s", desc.InitializationErrorString()); - auto cond_op = OpRegistry::CreateOp(desc, nullptr); + auto cond_op = OpRegistry::CreateOp(desc); return static_cast(cond_op.release()); }) .def("set_truenet", @@ -477,12 +486,10 @@ All parameter, weight, gradient are variables in Paddle. py::class_(m, "Executor") .def(py::init &>()) - .def("run", [](Executor &self, ProgramDescBind *program_bind, - Scope *scope, int block_id) { - self.Run(*program_bind->Proto(), scope, block_id); - }); + .def("run", &Executor::Run); m.def("unique_integer", UniqueIntegerGenerator); + m.def("init_gflags", InitGflags); m.def("is_compile_gpu", IsCompileGPU); m.def("set_feed_variable", framework::SetFeedVariable); @@ -493,6 +500,32 @@ All parameter, weight, gradient are variables in Paddle. BindVarDsec(m); BindOpDesc(m); + py::class_(m, "LodRankTable") + .def("items", [](framework::LoDRankTable &table) { + std::vector> res; + for (auto &item : table.items()) { + res.push_back({item.index, item.length}); + } + return res; + }); + + py::class_(m, "LoDTensorArray") + .def("__getitem__", + [](LoDTensorArray &self, size_t i) { return &self.at(i); }, + py::return_value_policy::reference) + .def("__len__", [](LoDTensorArray &self) { return self.size(); }) + .def("__setitem__", + [](LoDTensorArray &self, size_t i, const LoDTensor &t) { + PADDLE_ENFORCE_LT(i, self.size()); + self[i].ShareDataWith(t); + self[i].set_lod(t.lod()); + }) + .def("append", [](LoDTensorArray &self, const LoDTensor &t) { + self.emplace_back(); + self.back().ShareDataWith(t); + self.back().set_lod(t.lod()); + }); + m.def("op_support_gpu", OpSupportGPU); #ifdef PADDLE_WITH_CUDA m.def("get_cuda_device_count", platform::GetCUDADeviceCount); diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index a08716c5a559def54bb7b989f250b489f6a805a2..5bdf8c833522564e6b1027ca5dad8c0bb481cdc4 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -162,6 +162,7 @@ ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_GPU_ENV} ADD go/cmd/pserver/pserver /usr/bin/ ADD go/cmd/master/master /usr/bin/ +ADD paddle/pybind/print_operators_doc /usr/bin/ # default command shows the paddle version and exit CMD ["paddle", "version"] EOF diff --git a/paddle/scripts/docker/build_android.sh b/paddle/scripts/docker/build_android.sh index 11612ad4bed0afa8496087605afaefbd0420d5ce..6ef45d33d8c9e32e564555854c10a6fe15e4fd9f 100644 --- a/paddle/scripts/docker/build_android.sh +++ b/paddle/scripts/docker/build_android.sh @@ -4,6 +4,10 @@ set -xe if [ $ANDROID_ABI == "arm64-v8a" ]; then ANDROID_ARCH=arm64 + if [ $ANDROID_API -lt 21 ]; then + echo "Warning: arm64-v8a requires ANDROID_API >= 21." + ANDROID_API=21 + fi else # armeabi, armeabi-v7a ANDROID_ARCH=arm fi diff --git a/paddle/scripts/travis/build_doc.sh b/paddle/scripts/travis/build_doc.sh index dfcff38302703066e868c60e213f0f7cbc55a31e..973b2736e5ce2b733d52df4f5a270b296bca2cac 100755 --- a/paddle/scripts/travis/build_doc.sh +++ b/paddle/scripts/travis/build_doc.sh @@ -53,8 +53,8 @@ function deploy_docs() { set +e rm -rf ${DIR}/doc ${DIR}/doc_cn set -e - mv ../doc/cn/html ${DIR}/doc_cn - mv ../doc/en/html ${DIR}/doc + cp -r ../doc/cn/html ${DIR}/doc_cn + cp -r ../doc/en/html ${DIR}/doc git add . } diff --git a/paddle/trainer/MergeModel.cpp b/paddle/trainer/MergeModel.cpp index a70673ffec8812d86b9a0c13f15ef0b378dcf3ce..f3cfd9f97fea837e8f666f2eabee5a75659a4e42 100644 --- a/paddle/trainer/MergeModel.cpp +++ b/paddle/trainer/MergeModel.cpp @@ -27,6 +27,13 @@ using namespace paddle; // NOLINT using namespace std; // NOLINT int main(int argc, char** argv) { + if (FLAGS_model_dir.empty() || FLAGS_config_file.empty() || + FLAGS_model_file.empty()) { + LOG(INFO) << "Usage: ./paddle_merge_model --model_dir=pass-00000 " + "--config_file=config.py --model_file=out.paddle"; + return 0; + } + initMain(argc, argv); initPython(argc, argv); diff --git a/paddle/trainer/tests/CMakeLists.txt b/paddle/trainer/tests/CMakeLists.txt index 5ebbb99c94bce45d295ae0bf585f2cf864bfc4d4..f01ad4142d4fe7c7f7d7aac60d967ea114b93e56 100644 --- a/paddle/trainer/tests/CMakeLists.txt +++ b/paddle/trainer/tests/CMakeLists.txt @@ -37,22 +37,6 @@ add_test(NAME test_CompareTwoNets --config_file_a=trainer/tests/sample_trainer_config_qb_rnn.conf --config_file_b=trainer/tests/sample_trainer_config_rnn.conf WORKING_DIRECTORY ${PADDLE_SOURCE_DIR}/paddle/) -################ test_CompareMKLDNNandCPU ###################### -if(WITH_MKLDNN) - macro(gen_command VAR_NAME CONFIG_FILE) - set(${VAR_NAME} "${PADDLE_SOURCE_DIR}/paddle/.set_python_path.sh" "-d" "${PADDLE_SOURCE_DIR}/python/" - "${CMAKE_CURRENT_BINARY_DIR}/test_CompareMKLDNNandCPU --use_gpu=False" - "--config_file_a=trainer/tests/${CONFIG_FILE} --use_mkldnn_a=True" - "--config_file_b=trainer/tests/${CONFIG_FILE} --use_mkldnn_b=False" - "WORKING_DIRECTORY" "${PADDLE_SOURCE_DIR}/paddle/") - endmacro() - add_unittest_without_exec(test_CompareMKLDNNandCPU test_CompareTwoNets.cpp) - gen_command(compare_simple_net "sample_trainer_config_simple_net.conf") - gen_command(compare_branch_net "sample_trainer_config_branch_net.conf") - add_test(NAME test_CompareMKLDNNandCPU_simple_net COMMAND ${compare_simple_net}) - add_test(NAME test_CompareMKLDNNandCPU_branch_net COMMAND ${compare_branch_net}) -endif() - ############### test_CompareTwoOpts ################### add_unittest_without_exec(test_CompareTwoOpts test_CompareTwoOpts.cpp) diff --git a/paddle/trainer/tests/sample_trainer_config_branch_net.conf b/paddle/trainer/tests/sample_trainer_config_branch_net.conf deleted file mode 100644 index 3d8fb77a11958218091d2ee72e1d5a40ad1d9f5b..0000000000000000000000000000000000000000 --- a/paddle/trainer/tests/sample_trainer_config_branch_net.conf +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -################################### Data Configuration ################################### -TrainData(ProtoData(files = "trainer/tests/mnist.list")) -################################### Algorithm Configuration ################################### -settings(batch_size = 128, - learning_method = MomentumOptimizer(momentum=0.5, sparse=False)) -################################### Network Configuration ################################### -data = data_layer(name ="input", size=784) - -tmp = img_conv_layer(input=data, - num_channels=1, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=ReluActivation()) - -a1 = img_conv_layer(input=tmp, - filter_size=1, - num_filters=32, - padding=0, - shared_biases=True, - act=ReluActivation()) - -a2 = img_conv_layer(input=tmp, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=ReluActivation()) - -tmp = addto_layer(input=[a1, a2], - act=ReluActivation(), - bias_attr=False) - -tmp = img_pool_layer(input=tmp, - pool_size=3, - stride=2, - padding=1, - pool_type=AvgPooling()) - -b1 = img_conv_layer(input=tmp, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=ReluActivation()) - -b1 = img_pool_layer(input=b1, - pool_size=3, - stride=2, - padding=0, - pool_type=MaxPooling()) - -b2 = img_conv_layer(input=tmp, - filter_size=3, - num_filters=64, - padding=1, - shared_biases=True, - act=ReluActivation()) - -b2 = img_pool_layer(input=b2, - pool_size=5, - stride=2, - padding=1, - pool_type=MaxPooling()) - -tmp = concat_layer(input=[b1, b2]) - -tmp = img_pool_layer(input=tmp, - num_channels=96, - pool_size=3, - stride=2, - padding=1, - pool_type=MaxPooling()) - -tmp = img_conv_layer(input=tmp, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=LinearActivation(), - bias_attr=False) - -tmp = batch_norm_layer(input=tmp, - use_global_stats=False, - act=ReluActivation()) - -c1 = img_conv_layer(input=tmp, - filter_size=1, - num_filters=32, - padding=0, - shared_biases=True, - act=ReluActivation()) - -c2 = img_conv_layer(input=tmp, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=ReluActivation()) - -tmp = addto_layer(input=[c1, c2], - act=ReluActivation(), - bias_attr=False) - -tmp = fc_layer(input=tmp, size=64, - bias_attr=False, - act=TanhActivation()) - -output = fc_layer(input=tmp, size=10, - bias_attr=True, - act=SoftmaxActivation()) - -lbl = data_layer(name ="label", size=10) - -cost = classification_cost(input=output, label=lbl) -outputs(cost) diff --git a/paddle/trainer/tests/sample_trainer_config_simple_net.conf b/paddle/trainer/tests/sample_trainer_config_simple_net.conf deleted file mode 100644 index c615b5622b7e50b7aa99a9fcf9f63d7b4351417c..0000000000000000000000000000000000000000 --- a/paddle/trainer/tests/sample_trainer_config_simple_net.conf +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -################################### Data Configuration ################################### -TrainData(ProtoData(files = "trainer/tests/mnist.list")) -################################### Algorithm Configuration ################################### -settings(batch_size = 128, - learning_method = MomentumOptimizer(momentum=0.5, sparse=False)) -################################### Network Configuration ################################### -data = data_layer(name ="input", size=784) - -tmp = img_conv_layer(input=data, - num_channels=1, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=ReluActivation()) - -tmp = img_pool_layer(input=tmp, - pool_size=3, - stride=2, - padding=1, - pool_type=AvgPooling()) - -tmp = img_conv_layer(input=tmp, - filter_size=3, - num_filters=32, - padding=1, - shared_biases=True, - act=LinearActivation(), - bias_attr=False) - -tmp = batch_norm_layer(input=tmp, - use_global_stats=False, - act=ReluActivation()) - -tmp = img_pool_layer(input=tmp, - pool_size=3, - stride=2, - padding=1, - pool_type=MaxPooling()) - -tmp = fc_layer(input=tmp, size=64, - bias_attr=True, - act=ReluActivation()) - -output = fc_layer(input=tmp, size=10, - bias_attr=True, - act=SoftmaxActivation()) - -lbl = data_layer(name ="label", size=10) - -cost = classification_cost(input=output, label=lbl) -outputs(cost) diff --git a/paddle/trainer/tests/test_CompareTwoNets.cpp b/paddle/trainer/tests/test_CompareTwoNets.cpp index 307645d2c3d21d954371fcedb5f95a2536a0183e..94f65e545d116c802fb4877dc14f07aaaf83a4fb 100644 --- a/paddle/trainer/tests/test_CompareTwoNets.cpp +++ b/paddle/trainer/tests/test_CompareTwoNets.cpp @@ -26,15 +26,12 @@ DECLARE_int32(gpu_id); DECLARE_bool(local); DECLARE_bool(use_gpu); -DECLARE_bool(use_mkldnn); DECLARE_string(config); DECLARE_string(nics); DEFINE_string(config_file_a, "", "config of one network to compare"); DEFINE_string(config_file_b, "", "config of another network to compare"); -DEFINE_bool(use_mkldnn_a, false, "whether to use mkldnn to run config_file_a"); -DEFINE_bool(use_mkldnn_b, false, "whether to use mkldnn to run config_file_b"); DEFINE_bool(need_high_accuracy, false, "whether need to run in double accuracy"); @@ -131,12 +128,6 @@ void compareGradient(ComData& comDataA, ComData& comDataB) { matA.getWidth()); } - if (FLAGS_use_mkldnn_a || FLAGS_use_mkldnn_b) { - // some format of mkldnn parameter is different with cpu - // test_MKLDNN will check the parameters - return; - } - vector& parametersA = comDataA.parameters; vector& parametersB = comDataB.parameters; @@ -176,12 +167,10 @@ void compareGradient(ComData& comDataA, ComData& comDataB) { TEST(Trainer, create) { ComData dataA; - FLAGS_use_mkldnn = FLAGS_use_mkldnn_a; calcGradient(dataA, FLAGS_config_file_a); LOG(INFO) << "\n\nforwardBackward of Network A is finished\n\n"; ComData dataB; - FLAGS_use_mkldnn = FLAGS_use_mkldnn_b; calcGradient(dataB, FLAGS_config_file_b); LOG(INFO) << "\n\nforwardBackward of the Network B is finished\n\n"; diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 5d898d860cfc6dc26eaf5a81d8aed6d757ed5831..556bcd1d7e60c27fece43de666e9531ab4203414 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -27,3 +27,30 @@ foreach(filename ${proto_filenames}) endforeach() add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY}) + + +if (WITH_GOLANG) + add_custom_target(protoc-gen-go) + add_custom_command(TARGET protoc-gen-go + COMMAND go + ARGS "get" "-u" "github.com/golang/protobuf/protoc-gen-go") + + set(PROTO_GEN_GO) + file(GLOB proto_filenames . OptimizerConfig.proto) + foreach(filename ${proto_filenames}) + message(STATUS ${filename}) + get_filename_component(ABS_FIL ${filename} ABSOLUTE) + get_filename_component(FIL_WE ${filename} NAME_WE) + set(CUR_PROTO_GEN_GO + ${PADDLE_SOURCE_DIR}/paddle/go/proto/${FIL_WE}.pb.go) + set(PROTO_GEN_GO + ${CUR_PROTO_GEN_GO} + ${PROTO_GEN_GO}) + add_custom_command(OUTPUT ${CUR_PROTO_GEN_GO} + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS "--go_out=${PADDLE_SOURCE_DIR}/go/proto" + "-I" ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} protoc protoc-gen-go) + endforeach() + add_custom_target(gen_proto_go ALL DEPENDS ${PROTO_GEN_GO}) +endif() diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index e88e962cff5bbfcb8be1014dbaab85568d2625ff..0e65598485d8785b3f5b2f1bc7e87f377b35792e 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2775,9 +2775,15 @@ class NCELayer(LayerBase): @config_layer('addto') class AddToLayer(LayerBase): + layer_type = 'addto' + def __init__(self, name, inputs, bias=True, **xargs): + use_mkldnn = bool(int(g_command_config_args.get("use_mkldnn", 0))) + if self.layer_type == "mkldnn_addto": + config_assert(use_mkldnn, "mkldnn_addto only support MKLDNN") + self.layer_type = 'mkldnn_addto' if use_mkldnn else 'addto' super(AddToLayer, self).__init__( - name, 'addto', 0, inputs=inputs, **xargs) + name, self.layer_type, 0, inputs=inputs, **xargs) config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') if len(self.inputs) > 1: @@ -2796,6 +2802,11 @@ class AddToLayer(LayerBase): self.create_bias_parameter(bias, self.config.size) +@config_layer('mkldnn_addto') +class MKLDNNAddtoLayer(AddToLayer): + layer_type = 'mkldnn_addto' + + @config_layer('agent') class AgentLayer(LayerBase): def __init__(self, name, size, device=None): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index cc1b34df9e7cf8d17bafeb57624548de017066e9..6e8ac8838bf3012969bf4f597cdf4430693fe313 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1047,6 +1047,13 @@ def fc_layer(input, if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: + if "parameter_name" in param_attr.attr and len(input) > 1: + logger.fatal( + "When the name field of param_attr is manually specified " + "and the input is a list, the param_attr should also be a " + "list with each item being the param_attr for each input " + "item. If only one named param_attr is provided, all the " + "input items would share this parameter.") param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] assert isinstance(input, collections.Sequence) @@ -4877,6 +4884,13 @@ def selective_fc_layer(input, if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: + if "parameter_name" in param_attr.attr and len(input) > 1: + logger.fatal( + "When the name field of param_attr is manually specified " + "and the input is a list, the param_attr should also be a " + "list with each item being the param_attr for each input " + "item. If only one named param_attr is provided, all the " + "input items would share this parameter.") param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] assert isinstance(input, collections.Sequence) diff --git a/python/paddle/v2/dataset/imdb.py b/python/paddle/v2/dataset/imdb.py index 93dd3e8f7d3a569eaf56335f0f92bed04c0ee26c..cfc1c886e1389c15e3f803c341b6f62dd7b4bf41 100644 --- a/python/paddle/v2/dataset/imdb.py +++ b/python/paddle/v2/dataset/imdb.py @@ -116,7 +116,7 @@ def reader_creator(pos_pattern, neg_pattern, word_idx, buffer_size): yield [word_idx.get(w, UNK) for w in doc], i % 2 doc = qs[i % 2].get() - return reader() + return reader def train(word_idx): diff --git a/python/paddle/v2/framework/__init__.py b/python/paddle/v2/framework/__init__.py index c942373c667733f8aabe63026998a8915618130a..5df612bf3530c843c16b337f2b8f83445fcf39b5 100644 --- a/python/paddle/v2/framework/__init__.py +++ b/python/paddle/v2/framework/__init__.py @@ -1 +1,11 @@ +import sys +import core __all__ = ['proto'] +argv = [] +if core.is_compile_gpu(): + argv = list(sys.argv) + [ + "--tryfromenv=fraction_of_gpu_memory_to_use,use_pinned_memory" + ] +else: + argv = list(sys.argv) + ["--tryfromenv=use_pinned_memory"] +core.init_gflags(argv) diff --git a/python/paddle/v2/framework/backward.py b/python/paddle/v2/framework/backward.py index 6827792cb351243f926aeca5f37324dc987d6a79..678efd5d20585355a684bb2df16fdb57a69e0eeb 100644 --- a/python/paddle/v2/framework/backward.py +++ b/python/paddle/v2/framework/backward.py @@ -19,8 +19,20 @@ def append_backward_ops(loss, parameter_list=None, no_grad_set=None): :rtype: list[Variable] """ assert isinstance(loss, framework.Variable) - param_grad_map = loss.block.program.append_backward(loss, no_grad_set or - set()) + + if no_grad_set is None: + program = loss.block.program + assert isinstance(program, framework.Program) + no_grad_set = list() + for block in program.blocks: + assert isinstance(block, framework.Block) + for var in block.vars.itervalues(): + assert isinstance(var, framework.Variable) + if var.stop_gradient: + no_grad_set.append(var.name) + no_grad_set = set(no_grad_set) + + param_grad_map = loss.block.program.append_backward(loss, no_grad_set) if parameter_list is not None: parameters = parameter_list else: diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..254dd5f1a33eef17ad7a0117541255a4399ef23c --- /dev/null +++ b/python/paddle/v2/framework/evaluator.py @@ -0,0 +1,59 @@ +import paddle.v2.framework.op as op +import numpy as np +import paddle.v2.framework.core as core + + +def avg_accumulate(accumulated_var, per_eval, num_batches, place): + t = np.array(accumulated_var.get_tensor()) + t[0] += per_eval[0] + accumulated_var.get_tensor().set([t[0] / float(num_batches)], place) + + +class Evaluator(object): + def __init__(self, + scope, + operator='accuracy', + input='Inference', + label='Label', + output='Output', + place=core.CPUPlace()): + """ + create an evaluator for evaluating the inference. + NOTE: default run on CPUPlace(), running on GPUPlace doesn't improve performance much. + + :param scope: the scope instance contains the input. + :type scope: paddle.v2.framework.core.scope + :param operator: operator name for caculating the evaluation for each mini-batch. + :type operator: string + :param input: output variable name of forward network. + :type input: string + :param label: variable name of label + :type label: string + """ + self.scope = scope + self.place = place + self.output_name = output + self.num_batches = 0 + # create variable to store accumulated evaluator output + eval_name = ''.join([operator, "@Eval"]) + if scope.find_var(eval_name): + raise Exception("evaluator already exist in scope: %s" % eval_name) + self.accumulated_var = scope.var(eval_name) + t = self.accumulated_var.get_tensor() + t.set_dims((1, )) + t.set([0.0], place) + # self.accumulated_var = block.create_var(block, name=eval_name, shape=(1,)) + # self.accumulated_var.get_tensor().set([0.0]) + # create operator of evaluation + var_map = dict() # var name -> variable + var_map[input] = [input] + var_map[label] = [label] + var_map[output] = [output] + self.op = op.Operator(operator, **var_map) + + def evaluate(self, ctx, accumulator=avg_accumulate): + self.op.run(self.scope, ctx) + per_eval = np.array(self.scope.find_var(self.output_name).get_tensor()) + self.num_batches += 1 + accumulator(self.accumulated_var, per_eval, self.num_batches, + self.place) diff --git a/python/paddle/v2/framework/executor.py b/python/paddle/v2/framework/executor.py index d7d33903ff4f2244eb5365bf7f848c4390c8101b..8268d0d8f5126b00365d4e5e76cd98de4c47e670 100644 --- a/python/paddle/v2/framework/executor.py +++ b/python/paddle/v2/framework/executor.py @@ -62,7 +62,7 @@ class Executor(object): outputs={'Out': [fetch_var]}, attrs={'col': i}) - self.executor.run(program.desc, scope, 0) + self.executor.run(program.desc, scope, 0, True) return [ core.get_fetch_variable(scope, fetch_var_name, i) for i in xrange(len(fetch_list)) diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index 43101c9ddad76b7c1c322130dc0362a5c8ea4336..dd23c47961b6b27ffca32dc7fe496d715411c858 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -7,6 +7,11 @@ import copy __all__ = ['Block', 'Variable', 'Program', 'Operator'] +def unique_name(prefix): + uid = core.unique_integer(prefix) # unique during whole process. + return "_".join([prefix, str(uid)]) + + class Variable(object): def __init__(self, block, @@ -16,6 +21,7 @@ class Variable(object): dtype=None, lod_level=None, persistable=None, + stop_gradient=False, **kwargs): self.block = block @@ -84,6 +90,7 @@ class Variable(object): self.block.vars[name] = self self.op = None + self.stop_gradient = stop_gradient def __str__(self): protostr = self.desc.serialize_to_string() @@ -96,6 +103,10 @@ class Variable(object): def persistable(self): return self.desc.persistable() + @persistable.setter + def persistable(self, p): + self.desc.set_persistable(p) + @property def name(self): return self.desc.name() @@ -119,8 +130,9 @@ class Variable(object): @staticmethod def _unique_var_name_(): - uid = core.unique_integer() # unique during whole process. - return "_generated_var_%d" % uid + prefix = "_generated_var" + uid = core.unique_integer(prefix) # unique during whole process. + return "_".join([prefix, str(uid)]) @staticmethod def _convert_np_dtype_to_dtype_(np_dtype): @@ -263,7 +275,10 @@ class Operator(object): self.desc.set_attr(attr_name, attrs[attr_name]) self.desc.check_attrs() - no_kernel_op_set = {'feed', 'fetch', 'save', 'load'} + no_kernel_op_set = { + 'feed', 'fetch', 'save', 'load', 'recurrent', + 'rnn_memory_helper_grad' + } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) self.desc.infer_shape(self.block.desc) @@ -353,8 +368,8 @@ class Block(object): def create_var(self, *args, **kwargs): var = Variable(self, *args, **kwargs) - if 'init_attr' in kwargs: - self._prepend_initialize_ops_(var, kwargs['init_attr']) + if 'initializer' in kwargs: + kwargs['initializer'](var, self) return var def has_var(self, name): @@ -363,8 +378,8 @@ class Block(object): def create_parameter(self, *args, **kwargs): global_block = self.program.global_block() param = Parameter(global_block, *args, **kwargs) - if 'init_attr' in kwargs: - self._prepend_initialize_ops_(param, kwargs['init_attr']) + if 'initializer' in kwargs: + kwargs['initializer'](param, self) return param def append_op(self, *args, **kwargs): @@ -423,17 +438,6 @@ class Block(object): for index in range(len(self.ops)): assert self.ops[index].desc == ops_in_cpp[index] - def _prepend_initialize_ops_(self, param, init_attr): - op_type = init_attr['type'] - init_attr['shape'] = param.shape - init_attr['data_type'] = int(param.data_type) - op = self.prepend_op( - type=op_type, - inputs=None, - outputs={'Out': [param]}, - attrs=init_attr) - param.op = op - class Program(object): def __init__(self): @@ -548,5 +552,5 @@ class Parameter(Variable): # program is a global instance. -g_program = Program() -g_init_program = Program() +g_main_program = Program() +g_startup_program = Program() diff --git a/python/paddle/v2/framework/initializer.py b/python/paddle/v2/framework/initializer.py new file mode 100644 index 0000000000000000000000000000000000000000..98a87bfa86efb39f381b9f99b2b1f0d7ec7d9833 --- /dev/null +++ b/python/paddle/v2/framework/initializer.py @@ -0,0 +1,287 @@ +import paddle.v2.framework.framework as framework +import numpy as np + +__all__ = [ + 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', + 'XavierInitializer' +] + + +class Initializer(object): + """Base class for variable initializers + + Defines the common interface of variable initializers. + They add operations to the init program that are used + to initialize variables. Users should not use this class + directly, but need to use one of its implementations. + """ + + def __init_(self): + pass + + def __call__(self, param, block): + """Add corresponding initialization operations to the network + """ + raise NotImplementedError() + + def _compute_fans(self, var): + """Compute the fan_in and the fan_out for layers + + This method computes the fan_in and the fan_out + for neural network layers, if not specified. It is + not possible to perfectly estimate fan_in and fan_out. + This method will estimate it correctly for matrix multiply and + convolutions. + + Args: + var: variable for which fan_in and fan_out have to be computed + + Returns: + tuple of two integers (fan_in, fan_out) + """ + shape = var.shape + if not shape or len(shape) == 0: + fan_in = fan_out = 1 + elif len(shape) == 1: + fan_in = fan_out = shape[0] + elif len(shape) == 2: + # This is the case for simple matrix multiply + fan_in = shape[0] + fan_out = shape[1] + else: + # Assume this to be a convolutional kernel + # In PaddlePaddle, the shape of the kernel is like: + # [num_filters, num_filter_channels, ...] where the remaining + # dimensions are the filter_size + receptive_field_size = np.prod(shape[2:]) + fan_in = shape[1] * receptive_field_size + fan_out = shape[0] * receptive_field_size + + return (fan_in, fan_out) + + +class ConstantInitializer(Initializer): + """Implements the constant initializer + """ + + def __init__(self, value=0.0): + """Constructor for ConstantInitializer + + Args: + value: constant value to initialize the variable + """ + assert value is not None + super(ConstantInitializer, self).__init__() + self._value = value + + def __call__(self, var, block): + """Add constant initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + # Initialization Ops should be prepended and not appended + op = block.prepend_op( + type="fill_constant", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "value": self._value + }) + var.op = op + return op + + +class UniformInitializer(Initializer): + """Implements the random uniform distribution initializer + """ + + def __init__(self, low=-1.0, high=1.0, seed=0): + """Constructor for UniformInitializer + + Args: + low: lower boundary of the uniform distribution + high: upper boundary of the uniform distribution + seed: random seed + """ + assert low is not None + assert high is not None + assert high >= low + assert seed is not None + super(UniformInitializer, self).__init__() + self._low = low + self._high = high + self._seed = seed + + def __call__(self, var, block): + """Add uniform distribution initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + # Initialization Ops should be prepended and not appended + op = block.prepend_op( + type="uniform_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "min": self._low, + "max": self._high, + "seed": self._seed + }) + var.op = op + return op + + +class NormalInitializer(Initializer): + """Implements the random Normal(Gaussian) distribution initializer + """ + + def __init__(self, loc=0.0, scale=1.0, seed=0): + """Constructor for NormalInitializer + + Args: + loc: mean of the normal distribution + scale: standard deviation of the normal distribution + seed: random seed + """ + assert loc is not None + assert scale is not None + assert seed is not None + super(NormalInitializer, self).__init__() + self._mean = loc + self._std_dev = scale + self._seed = seed + + def __call__(self, var, block): + """Add normal distribution initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + # Initialization Ops should be prepended and not appended + op = block.prepend_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed + }) + var.op = op + return op + + +class XavierInitializer(Initializer): + """Implements the Xavier initializer + + This class implements the Xavier weight initializer from the paper + Understanding the difficulty of training deep feedforward neural + networks[1] by Xavier Glorot and Yoshua Bengio. + + This initializer is designed to keep the scale of the gradients + approximately same in all the layers. In case of Uniform distribution, + the range is [-x, x], where x = sqrt(6 / (fan_in + fan_out)). + In case of Normal distribution, the mean is 0 and the standard deviation + is sqrt(2/ (fan_in + fan_out)). + + References: + [1] Understanding the difficulty of training deep feedforward neural + networks. International conference on artificial intelligence and + statistics. + (http://proceedings.mlr.press/v9/glorot10a.html) + """ + + def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0): + """Constructor for XavierInitializer + + Args: + uniform: whether to use uniform or normal distribution + fan_in: fan_in for Xavier initialization. If None, it is + inferred from the variable. + fan_out: fan_out for Xavier initialization. If None, it is + inferred from the variable. + seed: random seed + + Note: It is recommended to set fan_in and fan_out to None for + most cases. + """ + assert uniform is not None + assert seed is not None + super(XavierInitializer, self).__init__() + self._uniform = uniform + self._fan_in = fan_in + self._fan_out = fan_out + self._seed = seed + + def __call__(self, var, block): + """Add xavier initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + f_in, f_out = self._compute_fans(var) + + # If fan_in and fan_out are passed, use them + fan_in = f_in if self._fan_in is None else self._fan_in + fan_out = f_out if self._fan_out is None else self._fan_out + + if self._uniform: + limit = np.sqrt(6.0 / float(fan_in + fan_out)) + op = block.prepend_op( + type="uniform_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "min": -limit, + "max": limit, + "seed": self._seed + }) + + else: + std = np.sqrt(2.0 / float(fan_in + fan_out)) + op = block.prepend_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "mean": 0.0, + "std": std, + "seed": self._seed + }) + var.op = op + return op diff --git a/python/paddle/v2/framework/io.py b/python/paddle/v2/framework/io.py index f3ba719bde086f696a27b806228a8c97466a681e..5c247904a330e25b1a9f53db431947840db3f615 100644 --- a/python/paddle/v2/framework/io.py +++ b/python/paddle/v2/framework/io.py @@ -1,7 +1,7 @@ import os import cPickle as pickle -from paddle.v2.framework.framework import Program, Parameter, g_program, \ +from paddle.v2.framework.framework import Program, Parameter, g_main_program, \ Variable __all__ = [ @@ -29,13 +29,13 @@ def _clone_var_in_block_(block, var): persistable=True) -def save_vars(executor, dirname, program=None, vars=None, predicate=None): +def save_vars(executor, dirname, main_program=None, vars=None, predicate=None): """ Save variables to directory by executor. :param executor: executor that save variable :param dirname: directory path - :param program: program. If vars is None, then filter all variables in this + :param main_program: program. If vars is None, then filter all variables in this program which fit `predicate`. Default g_program. :param predicate: The Predicate describes a callable that returns a variable as a bool. If it returns true, the variables will be saved. @@ -44,15 +44,15 @@ def save_vars(executor, dirname, program=None, vars=None, predicate=None): :return: None """ if vars is None: - if program is None: - program = g_program - if not isinstance(program, Program): + if main_program is None: + main_program = g_main_program + if not isinstance(main_program, Program): raise TypeError("program should be as Program type or None") save_vars( executor, dirname=dirname, - vars=filter(predicate, program.list_vars())) + vars=filter(predicate, main_program.list_vars())) else: save_program = Program() save_block = save_program.global_block() @@ -66,37 +66,37 @@ def save_vars(executor, dirname, program=None, vars=None, predicate=None): executor.run(save_program) -def save_params(executor, dirname, program=None): +def save_params(executor, dirname, main_program=None): """ Save all parameters to directory with executor. """ save_vars( executor, dirname=dirname, - program=program, + main_program=main_program, vars=None, predicate=is_parameter) -def save_persistables(executor, dirname, program=None): +def save_persistables(executor, dirname, main_program=None): """ Save all persistables to directory with executor. """ save_vars( executor, dirname=dirname, - program=program, + main_program=main_program, vars=None, predicate=is_persistable) -def load_vars(executor, dirname, program=None, vars=None, predicate=None): +def load_vars(executor, dirname, main_program=None, vars=None, predicate=None): """ Load variables from directory by executor. :param executor: executor that save variable :param dirname: directory path - :param program: program. If vars is None, then filter all variables in this + :param main_program: program. If vars is None, then filter all variables in this program which fit `predicate`. Default g_program. :param predicate: The Predicate describes a callable that returns a variable as a bool. If it returns true, the variables will be loaded. @@ -105,15 +105,15 @@ def load_vars(executor, dirname, program=None, vars=None, predicate=None): :return: None """ if vars is None: - if program is None: - program = g_program - if not isinstance(program, Program): + if main_program is None: + main_program = g_main_program + if not isinstance(main_program, Program): raise TypeError("program's type should be Program") load_vars( executor, dirname=dirname, - vars=filter(predicate, program.list_vars())) + vars=filter(predicate, main_program.list_vars())) else: load_prog = Program() load_block = load_prog.global_block() @@ -129,27 +129,33 @@ def load_vars(executor, dirname, program=None, vars=None, predicate=None): executor.run(load_prog) -def load_params(executor, dirname, program=None): +def load_params(executor, dirname, main_program=None): """ load all parameters from directory by executor. """ load_vars( - executor, dirname=dirname, program=program, predicate=is_parameter) + executor, + dirname=dirname, + main_program=main_program, + predicate=is_parameter) -def load_persistables(executor, dirname, program=None): +def load_persistables(executor, dirname, main_program=None): """ load all persistables from directory by executor. """ load_vars( - executor, dirname=dirname, program=program, predicate=is_persistable) + executor, + dirname=dirname, + main_program=main_program, + predicate=is_persistable) def save_inference_model(dirname, feeded_var_names, target_vars, executor, - program=None): + main_program=None): """ Build a model especially for inference, and save it to directory by the executor. @@ -158,20 +164,20 @@ def save_inference_model(dirname, :param feeded_var_names: Names of variables that need to be feeded data during inference :param target_vars: Variables from which we can get inference results. :param executor: executor that save inference model - :param program: original program, which will be pruned to build the inference model. + :param main_program: original program, which will be pruned to build the inference model. Default g_program. :return: None """ - if program is None: - program = g_program + if main_program is None: + main_program = g_main_program if not isinstance(target_vars, list): target_vars = [target_vars] if not os.path.isdir(dirname): os.makedirs(dirname) - pruned_program = program.prune(target_vars) + pruned_program = main_program.prune(target_vars) fetch_var_names = [v.name for v in target_vars] model_file_name = dirname + "/__model__" @@ -182,10 +188,10 @@ def save_inference_model(dirname, "fetch_var_names": fetch_var_names }, f, -1) - save_params(executor, dirname, program) + save_params(executor, dirname, main_program) -def load_persistables_if_exist(executor, dirname, program=None): +def load_persistables_if_exist(executor, dirname, main_program=None): filenames = next(os.walk(dirname))[2] filenames = set(filenames) @@ -198,7 +204,7 @@ def load_persistables_if_exist(executor, dirname, program=None): load_vars( executor, dirname, - program=program, + main_program=main_program, vars=None, predicate=_is_presistable_and_exist_) diff --git a/python/paddle/v2/framework/layer_helper.py b/python/paddle/v2/framework/layer_helper.py index 1f72c9bc7b0ceda1dd954703fcc10c77a3e5ed25..c38346b79fecfb2f82a60b360c505da16ecdf3c0 100644 --- a/python/paddle/v2/framework/layer_helper.py +++ b/python/paddle/v2/framework/layer_helper.py @@ -1,15 +1,10 @@ import copy import itertools -import paddle.v2.framework.core as core - -from paddle.v2.framework.framework import Variable, g_program, \ - g_init_program - - -def unique_name(prefix): - uid = core.unique_integer() # unique during whole process. - return "_".join([prefix, str(uid)]) +from paddle.v2.framework.framework import Variable, g_main_program, \ + g_startup_program, unique_name, Program +from paddle.v2.framework.initializer import ConstantInitializer, \ + UniformInitializer class LayerHelper(object): @@ -25,23 +20,23 @@ class LayerHelper(object): return self.kwargs['name'] @property - def program(self): - prog = self.kwargs.get('program', None) + def main_program(self): + prog = self.kwargs.get('main_program', None) if prog is None: - return g_program + return g_main_program else: return prog @property - def init_program(self): - prog = self.kwargs.get('init_program', None) + def startup_program(self): + prog = self.kwargs.get('startup_program', None) if prog is None: - return g_init_program + return g_startup_program else: return prog def append_op(self, *args, **kwargs): - return self.program.current_block().append_op(*args, **kwargs) + return self.main_program.current_block().append_op(*args, **kwargs) def multiple_input(self, input_param_name='input'): inputs = self.kwargs.get(input_param_name, []) @@ -66,14 +61,7 @@ class LayerHelper(object): @property def param_attr(self): - default = { - 'name': None, - 'init_attr': { - 'type': 'uniform_random', - 'min': -1.0, - 'max': 1.0 - } - } + default = {'name': None, 'initializer': UniformInitializer()} actual = self.kwargs.get('param_attr', None) if actual is None: actual = default @@ -83,13 +71,7 @@ class LayerHelper(object): return actual def bias_attr(self): - default = { - 'name': None, - 'init_attr': { - 'type': 'fill_constant', - 'value': 0.0 - } - } + default = {'name': None, 'initializer': ConstantInitializer()} bias_attr = self.kwargs.get('bias_attr', None) if bias_attr is True: bias_attr = default @@ -130,31 +112,60 @@ class LayerHelper(object): raise ValueError("Data Type mismatch") return dtype - def create_parameter(self, attr, shape, dtype, suffix='w'): + def create_parameter(self, attr, shape, dtype, suffix='w', + initializer=None): # Deepcopy the attr so that parameters can be shared in program attr_copy = copy.deepcopy(attr) + if initializer is not None: + attr_copy['initializer'] = initializer if attr_copy['name'] is None: attr_copy['name'] = unique_name(".".join([self.name, suffix])) - self.init_program.global_block().create_parameter( + self.startup_program.global_block().create_parameter( dtype=dtype, shape=shape, **attr_copy) - return self.program.global_block().create_parameter( + return self.main_program.global_block().create_parameter( name=attr_copy['name'], dtype=dtype, shape=shape) def create_tmp_variable(self, dtype): - return self.program.current_block().create_var( + return self.main_program.current_block().create_var( name=unique_name(".".join([self.name, 'tmp'])), dtype=dtype, persistable=False) def create_variable(self, *args, **kwargs): - return self.program.current_block().create_var(*args, **kwargs) - - def create_global_variable(self, *args, **kwargs): - return self.program.global_block().create_var( - *args, persistable=False, **kwargs) - - def append_bias_op(self, input_var): - size = list(input_var.shape[1:]) + return self.main_program.current_block().create_var(*args, **kwargs) + + def create_global_variable(self, persistable=False, *args, **kwargs): + return self.main_program.global_block().create_var( + *args, persistable=persistable, **kwargs) + + def set_variable_initializer(self, var, initializer): + assert isinstance(var, Variable) + self.startup_program.global_block().create_var( + name=var.name, + type=var.type, + dtype=var.data_type, + shape=var.shape, + persistable=True, + initializer=initializer) + + def append_bias_op(self, input_var, num_flatten_dims=None): + """ + Append bias operator and return its output. If the user does not set + bias_attr, append_bias_op will return input_var + + :param input_var: the input variable. The len(input_var.shape) is larger + or equal than 2. + :param num_flatten_dims: The input tensor will be flatten as a matrix + when adding bias. + `matrix.shape = product(input_var.shape[0:num_flatten_dims]), product( + input_var.shape[num_flatten_dims:])` + """ + if num_flatten_dims is None: + num_flatten_dims = self.kwargs.get('num_flatten_dims', None) + if num_flatten_dims is None: + num_flatten_dims = 1 + + size = list(input_var.shape[num_flatten_dims:]) bias_attr = self.bias_attr() if not bias_attr: return input_var diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py index 041a3b2c0b03c8171c2af9d856b33f461bb486c1..0739b2d2e2964f7f3565baf068a80c89938d193a 100644 --- a/python/paddle/v2/framework/layers.py +++ b/python/paddle/v2/framework/layers.py @@ -1,11 +1,13 @@ -from paddle.v2.framework.layer_helper import LayerHelper, unique_name import paddle.v2.framework.core as core -from paddle.v2.framework.framework import OpProtoHolder, Variable, Program +from paddle.v2.framework.framework import OpProtoHolder, Variable, Program, Operator +from paddle.v2.framework.initializer import ConstantInitializer, NormalInitializer +from paddle.v2.framework.layer_helper import LayerHelper, unique_name import re __all__ = [ 'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat', - 'StaticRNN', 'cast' + 'StaticRNN', 'cast', 'sequence_conv', 'sequence_pool', 'sums', 'cos_sim', + 'batch_norm', 'accuracy' ] @@ -16,8 +18,8 @@ def fc(input, name=None, act=None, num_flatten_dims=1, - program=None, - init_program=None): + main_program=None, + startup_program=None): # create helper helper = LayerHelper('fc', **locals()) @@ -30,7 +32,6 @@ def fc(input, param_shape = [ reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) ] + [size] - w = helper.create_parameter( attr=param_attr, shape=param_shape, dtype=dtype) tmp = helper.create_tmp_variable(dtype) @@ -63,8 +64,8 @@ def embedding(input, data_type='float32', is_sparse=False, param_attr=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): helper = LayerHelper('embedding', **locals()) w = helper.create_parameter( attr=helper.param_attr, shape=size, dtype=data_type) @@ -83,13 +84,22 @@ def data(name, data_type='float32', type=core.VarDesc.VarType.LOD_TENSOR, append_batch_size=True, - program=None, - init_program=None): + main_program=None, + startup_program=None): helper = LayerHelper('data', **locals()) + shape = list(shape) + for i in xrange(len(shape)): + if shape[i] is None: + shape[i] = -1 + append_batch_size = False + elif shape[i] < 0: + append_batch_size = False + if append_batch_size: shape = [-1] + shape # append batch size as -1 + return helper.create_global_variable( - name=name, shape=shape, dtype=data_type, type=type) + name=name, shape=shape, dtype=data_type, type=type, stop_gradient=True) def _convert_(name): @@ -150,7 +160,7 @@ def _create_op_func_(op_type): outputs[name] = [helper.create_tmp_variable(dtype=dtype)] helper.append_op( type=op_type, inputs=inputs, outputs=outputs, attrs=kwargs) - return out + return helper.append_activation(out) func.__name__ = op_type globals()[op_type] = func @@ -160,11 +170,15 @@ def _create_op_func_(op_type): _create_op_func_('mean') _create_op_func_('mul') +_create_op_func_('elementwise_add') _create_op_func_('dropout') _create_op_func_('reshape') +_create_op_func_('elementwise_add') +_create_op_func_('sigmoid') +_create_op_func_('scale') -def cast(x, data_type, program=None): +def cast(x, data_type, main_program=None): helper = LayerHelper('cast', **locals()) out = helper.create_tmp_variable(dtype=data_type) helper.append_op( @@ -176,11 +190,9 @@ def cast(x, data_type, program=None): return out -def concat(input, axis, program=None, init_program=None): +def concat(input, axis, main_program=None, startup_program=None): helper = LayerHelper('concat', **locals()) - if not isinstance(input, list) and not isinstance(input, tuple): - input = [input] - out = helper.create_tmp_variable(dtype=input[0].data_type) + out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op( type='concat', inputs={'X': input}, @@ -189,6 +201,28 @@ def concat(input, axis, program=None, init_program=None): return out +def sums(input, main_program=None, startup_program=None): + helper = LayerHelper('sum', **locals()) + out = helper.create_tmp_variable(dtype=helper.input_dtype()) + helper.append_op(type='sum', inputs={'X': input}, outputs={'Out': out}) + return out + + +def cos_sim(X, Y, **kwargs): + helper = LayerHelper('cos_sim', **kwargs) + out = helper.create_tmp_variable(dtype=X.data_type) + xnorm = helper.create_tmp_variable(dtype=X.data_type) + ynorm = helper.create_tmp_variable(dtype=X.data_type) + helper.append_op( + type='cos_sim', + inputs={'X': [X], + 'Y': [Y]}, + outputs={'Out': [out], + 'XNorm': [xnorm], + 'YNorm': [ynorm]}) + return out + + def cross_entropy(input, label, **kwargs): helper = LayerHelper('cross_entropy', **kwargs) out = helper.create_tmp_variable(dtype=input.data_type) @@ -212,13 +246,71 @@ def square_error_cost(input, label, **kwargs): square_out = helper.create_tmp_variable(dtype=input.data_type) helper.append_op( - type='pow', - inputs={'X': [minus_out]}, - outputs={'Y': [square_out]}, - attrs={'factor': 2.0}) + type='square', inputs={'X': [minus_out]}, outputs={'Y': [square_out]}) return square_out +def accuracy(input, label, k=1, **kwargs): + helper = LayerHelper("accuracy", **kwargs) + topk_out = helper.create_tmp_variable(dtype=input.data_type) + topk_indices = helper.create_tmp_variable(dtype="int64") + helper.append_op( + type="top_k", + inputs={"X": [input]}, + outputs={"Out": [topk_out], + "Indices": [topk_indices]}, + attrs={"k": k}) + acc_out_dtype = kwargs.get("out_dtype", "float32") + acc_out = helper.create_tmp_variable(dtype=acc_out_dtype) + helper.append_op( + type="accuracy", + inputs={ + "Out": [topk_out], + "Indices": [topk_indices], + "Label": [label] + }, + outputs={"Accuracy": [acc_out]}) + return acc_out + + +def sequence_conv(input, + num_filters, + filter_size=3, + filter_stride=1, + act=None, + padding=None, + bias_attr=None, + param_attr=None, + main_program=None, + startup_program=None): + # FIXME(dzh) : want to unify the argument of python layer + # function. So we ignore some unecessary attributes. + # such as, padding_trainable, context_start. + + helper = LayerHelper('sequence_conv', **locals()) + dtype = helper.input_dtype() + + filter_shape = [filter_size * input.shape[1], num_filters] + filter = helper.create_parameter( + attr=helper.param_attr, shape=filter_shape, dtype=dtype) + pre_bias = helper.create_tmp_variable(dtype) + + helper.append_op( + type='sequence_conv', + inputs={ + 'X': [input], + 'Filter': [filter], + }, + outputs={"Out": pre_bias}, + attrs={ + 'contextStride': filter_stride, + 'contextStart': -int(filter_size / 2), + 'contextLength': filter_size + }) + pre_act = helper.append_bias_op(pre_bias) + return helper.append_activation(pre_act) + + def conv2d(input, num_filters, name=None, @@ -229,8 +321,8 @@ def conv2d(input, padding=None, bias_attr=None, param_attr=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): helper = LayerHelper('conv2d', **locals()) dtype = helper.input_dtype() @@ -251,8 +343,13 @@ def conv2d(input, input_shape = input.shape filter_shape = [num_filters, num_filter_channels] + filter_size + + std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 filter = helper.create_parameter( - attr=helper.param_attr, shape=filter_shape, dtype=dtype) + attr=helper.param_attr, + shape=filter_shape, + dtype=dtype, + initializer=NormalInitializer(0.0, std, 0)) pre_bias = helper.create_tmp_variable(dtype) helper.append_op( @@ -266,19 +363,33 @@ def conv2d(input, 'paddings': padding, 'groups': groups}) - pre_act = helper.append_bias_op(pre_bias) + pre_act = helper.append_bias_op(pre_bias, 1) return helper.append_activation(pre_act) +def sequence_pool(input, pool_type, **kwargs): + helper = LayerHelper('sequence_pool', input=input, **kwargs) + dtype = helper.input_dtype() + pool_out = helper.create_tmp_variable(dtype) + + helper.append_op( + type="sequence_pool", + inputs={"X": [input]}, + outputs={"Out": [pool_out]}, + attrs={"pooltype": pool_type.upper()}) + + return pool_out + + def pool2d(input, pool_size, pool_type, pool_stride=[1, 1], pool_padding=[0, 0], global_pooling=False, - program=None, - init_program=None): + main_program=None, + startup_program=None): if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", @@ -290,7 +401,7 @@ def pool2d(input, if isinstance(pool_padding, int): pool_padding = [pool_padding, pool_padding] - helper = LayerHelper('conv2d', **locals()) + helper = LayerHelper('pool2d', **locals()) dtype = helper.input_dtype() pool_out = helper.create_tmp_variable(dtype) @@ -313,12 +424,12 @@ def batch_norm(input, act=None, is_test=False, momentum=0.9, - epsilon=1e05, + epsilon=1e-05, param_attr=None, bias_attr=None, data_layout='NCHW', - program=None, - init_program=None): + main_program=None, + startup_program=None): helper = LayerHelper('batch_norm', **locals()) dtype = helper.input_dtype() @@ -331,40 +442,29 @@ def batch_norm(input, else: raise ValueError("unsupported data layout:" + data_layout) - def get_init_attr(value): - if not isinstance(value, float): - raise ValueError("attr value should be a float") - return {'type': 'fill_constant', 'value': value} - - def prepend_init_op(var, init_attr): - assert isinstance(var, Variable) - op_type = init_attr['type'] - init_attr['shape'] = var.shape - init_attr['data_type'] = int(var.data_type) - op = var.block.prepend_op( - type=op_type, inputs=None, outputs={'Out': [var]}, attrs=init_attr) - return op - - def create_persistable_var(dtype, shape, init_attr=None): - name = unique_name(".".join([helper.name, "xxxx"])) - var = init_program.global_block().create_var( - dtype=dtype, shape=shape, name=name, persistable=True) - if 'init_attr' is not None: - prepend_init_op(var, init_attr) - return program.global_block().create_var( - name=name, dtype=dtype, shape=shape, persistable=True) - param_shape = [channel_num] # create parameter scale = helper.create_parameter( - attr=helper.param_attr, shape=param_shape, dtype=dtype) + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + initializer=ConstantInitializer(1.0)) bias = helper.create_parameter( - attr=helper.param_attr, shape=param_shape, dtype=dtype) + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + initializer=ConstantInitializer(0.0)) - # create input - mean = create_persistable_var(dtype, param_shape, get_init_attr(0.0)) - variance = create_persistable_var(dtype, param_shape, get_init_attr(1.0)) + mean = helper.create_global_variable( + dtype=input.data_type, shape=param_shape, persistable=True) + helper.set_variable_initializer( + var=mean, initializer=ConstantInitializer(0.0)) + + variance = helper.create_global_variable( + dtype=input.data_type, shape=param_shape, persistable=True) + helper.set_variable_initializer( + var=variance, initializer=ConstantInitializer(1.0)) # create output # mean and mean_out share the same memory @@ -405,16 +505,16 @@ class BlockGuard(object): keyword. """ - def __init__(self, program): - if not isinstance(program, Program): + def __init__(self, main_program): + if not isinstance(main_program, Program): raise TypeError("BlockGuard takes a program") - self.program = program + self.main_program = main_program def __enter__(self): - self.program.create_block() + self.main_program.create_block() def __exit__(self, exc_type, exc_val, exc_tb): - self.program.rollback() + self.main_program.rollback() if exc_type is not None: return False # re-raise exception return True @@ -424,7 +524,7 @@ class StaticRNNGuard(BlockGuard): def __init__(self, rnn): if not isinstance(rnn, StaticRNN): raise TypeError("StaticRNNGuard takes an StaticRNN") - super(StaticRNNGuard, self).__init__(rnn.helper.program) + super(StaticRNNGuard, self).__init__(rnn.helper.main_program) self.rnn = rnn def __enter__(self): @@ -432,6 +532,8 @@ class StaticRNNGuard(BlockGuard): return super(StaticRNNGuard, self).__enter__() def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + return False self.rnn.status = StaticRNN.AFTER_RNN_BLOCK self.rnn.complete_rnn_op() return super(StaticRNNGuard, self).__exit__(exc_type, exc_val, exc_tb) @@ -458,8 +560,9 @@ class StaticRNN(object): IN_RNN_BLOCK = 1 AFTER_RNN_BLOCK = 2 - def __init__(self, name=None, program=None): - self.helper = LayerHelper("static_rnn", name=name, program=program) + def __init__(self, name=None, main_program=None): + self.helper = LayerHelper( + "static_rnn", name=name, main_program=main_program) self.memories = {} # memory map, from pre_mem.name --> MemoryLink self.inputs = [] # input variable list in current block self.outputs = [] # output variable list in parent block @@ -491,7 +594,7 @@ class StaticRNN(object): outputs={'Out': [boot_var]}, attrs={ 'value': init_value, - 'shape': boot_var.shape, + 'shape': [40] + list(boot_var.shape[1:]), 'data_type': boot_var.data_type }) @@ -510,14 +613,14 @@ class StaticRNN(object): if not isinstance(x, Variable): raise TypeError("step input takes a Variable") if self.seq_len is None: - self.seq_len = x.shape[1] - elif self.seq_len != x.shape[1]: + self.seq_len = x.shape[0] + elif self.seq_len != x.shape[0]: raise ValueError("Static RNN only take fix seq_len input") ipt = self.helper.create_variable( name=x.name, dtype=x.data_type, - shape=[-1] + list(x.shape[2:]), + shape=list(x.shape[1:]), type=x.type) self.inputs.append(ipt) return ipt @@ -527,10 +630,17 @@ class StaticRNN(object): if not isinstance(o, Variable): raise TypeError("step output takes a Variable") + tmp_o = self.helper.create_tmp_variable(dtype=o.data_type) + self.helper.append_op( + type='rnn_memory_helper', + inputs={'X': [o]}, + outputs={'Out': tmp_o}, + attrs={'data_type': o.data_type}) + out_var = self.parent_block().create_var( - name=o.name, - shape=[-1, self.seq_len] + list(o.shape[1:]), - dtype=o.data_type) + name=tmp_o.name, + shape=[self.seq_len] + list(tmp_o.shape), + dtype=tmp_o.data_type) self.outputs.append(out_var) @@ -544,7 +654,7 @@ class StaticRNN(object): self.memories[mem.name].mem = var def parent_block(self): - prog = self.helper.program + prog = self.helper.main_program parent_idx = prog.current_block().parent_idx assert parent_idx >= 0 parent_block = prog.block(parent_idx) @@ -561,6 +671,81 @@ class StaticRNN(object): return self.outputs def complete_rnn_op(self): - # TODO(yuyang18): Create RNN Op here. - # Implement this method after RNN op complete. - pass + main_program = self.helper.main_program + rnn_block = main_program.current_block() + parent_block = self.parent_block() + + local_inputs = set() + + for op in rnn_block.ops: + assert isinstance(op, Operator) + for oname in op.output_names: + for out_var_name in op.output(oname): + local_inputs.add(out_var_name) + + for var in self.inputs: + local_inputs.add(var.name) + for m in self.memories: + local_inputs.add(m) + + params = list() + for op in rnn_block.ops: + assert isinstance(op, Operator) + for iname in op.input_names: + for in_var_name in op.input(iname): + if in_var_name not in local_inputs: + params.append(in_var_name) + + parameters = [parent_block.var(name) for name in params] + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + + inlinks = [parent_block.var(i.name) for i in self.inputs] + outlinks = self.outputs + + boot_memories = [] + pre_memories = [] + memories = [] + for _, mem in self.memories.iteritems(): + boot_memories.append(mem.init) + pre_memories.append(mem.pre_mem.name) + mem_var = rnn_block.var(mem.mem.name) + assert isinstance(mem_var, Variable) + new_mem = self.helper.create_tmp_variable(dtype=mem_var.data_type) + + rnn_block.append_op( + type='rnn_memory_helper', + inputs={'X': [mem_var]}, + outputs={'Out': [new_mem]}, + attrs={'data_type': mem_var.data_type}) + + memories.append(new_mem.name) + + parent_block.append_op( + type='recurrent', + inputs={ + 'inputs': inlinks, + 'initial_states': boot_memories, + 'parameters': parameters + }, + outputs={'outputs': outlinks, + 'step_scopes': [step_scope]}, + attrs={ + 'ex_states': pre_memories, + 'states': memories, + 'step_block': rnn_block + }) + + +def lod_rank_table(x, level=0, main_program=None): + helper = LayerHelper("lod_rank_table", **locals()) + table = helper.create_variable( + type=core.VarDesc.VarType.LOD_RANK_TABLE, + name=unique_name("lod_rank_table")) + helper.append_op( + type='lod_rank_table', + inputs={'X': x}, + outputs={'Out': table}, + attrs={'level': level}) + return table diff --git a/python/paddle/v2/framework/net_drawer.py b/python/paddle/v2/framework/net_drawer.py new file mode 100644 index 0000000000000000000000000000000000000000..045e267c253e2485e75df3fb95cc0e591ee29ea5 --- /dev/null +++ b/python/paddle/v2/framework/net_drawer.py @@ -0,0 +1,109 @@ +import argparse +import json +import logging +from collections import defaultdict + +import paddle.v2.framework.core as core +import paddle.v2.framework.proto.framework_pb2 as framework_pb2 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +try: + from graphviz import Digraph +except ImportError: + logger.info( + 'Cannot import graphviz, which is required for drawing a network. This ' + 'can usually be installed in python with "pip install graphviz". Also, ' + 'pydot requires graphviz to convert dot files to pdf: in ubuntu, this ' + 'can usually be installed with "sudo apt-get install graphviz".') + print('net_drawer will not run correctly. Please install the correct ' + 'dependencies.') + exit(0) + +OP_STYLE = { + 'shape': 'oval', + 'color': '#0F9D58', + 'style': 'filled', + 'fontcolor': '#FFFFFF' +} + +VAR_STYLE = {} + +GRAPH_STYLE = {"rankdir": "TB", } + +GRAPH_ID = 0 + + +def unique_id(): + def generator(): + GRAPH_ID += 1 + return GRAPH_ID + + return generator + + +def draw_node(op): + node = OP_STYLE + node["name"] = op.type + node["label"] = op.type + return node + + +def draw_edge(var_parent, op, var, arg): + edge = VAR_STYLE + edge["label"] = "%s(%s)" % (var.parameter, arg) + edge["head_name"] = op.type + edge["tail_name"] = var_parent[arg] + return edge + + +def parse_graph(program, graph, var_dict, **kwargs): + + # fill the known variables + for block in program.blocks: + for var in block.vars: + if not var_dict.has_key(var): + var_dict[var] = "Feed" + + proto = framework_pb2.ProgramDesc.FromString( + program.desc.serialize_to_string()) + for block in proto.blocks: + for op in block.ops: + graph.node(**draw_node(op)) + for o in op.outputs: + for arg in o.arguments: + var_dict[arg] = op.type + for e in op.inputs: + for arg in e.arguments: + if var_dict.has_key(arg): + graph.edge(**draw_edge(var_dict, op, e, arg)) + + +def draw_graph(startup_program, main_program, **kwargs): + if kwargs.has_key("graph_attr"): + GRAPH_STYLE.update(kwargs[graph_attr]) + if kwargs.has_key("node_attr"): + OP_STYLE.update(kwargs[node_attr]) + if kwargs.has_key("edge_attr"): + VAR_STYLE.update(kwargs[edge_attr]) + + graph_id = unique_id() + filename = kwargs.get("filename") + if filename == None: + filename = str(graph_id) + ".gv" + g = Digraph( + name=str(graph_id), + filename=filename, + graph_attr=GRAPH_STYLE, + node_attr=OP_STYLE, + edge_attr=VAR_STYLE, + **kwargs) + + var_dict = {} + parse_graph(startup_program, g, var_dict) + parse_graph(main_program, g, var_dict) + + if filename != None: + g.save() + return g diff --git a/python/paddle/v2/framework/nets.py b/python/paddle/v2/framework/nets.py index 803534fa391c49d646c5d98a442d35d06b98603e..725d2fa7f5e7a862eea0ef9172a9e63858ebd0dd 100644 --- a/python/paddle/v2/framework/nets.py +++ b/python/paddle/v2/framework/nets.py @@ -1,30 +1,32 @@ import paddle.v2.framework.layers as layers +__all__ = ["simple_img_conv_pool", "sequence_conv_pool"] + def simple_img_conv_pool(input, - filter_size, num_filters, + filter_size, pool_size, pool_stride, act, pool_type='max', - program=None, - init_program=None): + main_program=None, + startup_program=None): conv_out = layers.conv2d( input=input, num_filters=num_filters, filter_size=filter_size, act=act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) pool_out = layers.pool2d( input=conv_out, pool_size=pool_size, pool_type=pool_type, pool_stride=pool_stride, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return pool_out @@ -38,14 +40,14 @@ def img_conv_group(input, conv_batchnorm_drop_rate=None, pool_stride=1, pool_type=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): """ Image Convolution Group, Used for vgg net. """ tmp = input assert isinstance(conv_num_filter, list) or \ - isinstance(conv_num_filter, tuple) + isinstance(conv_num_filter, tuple) def __extend_list__(obj): if not hasattr(obj, '__len__'): @@ -69,28 +71,51 @@ def img_conv_group(input, filter_size=conv_filter_size[i], padding=conv_padding[i], act=local_conv_act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) if conv_with_batchnorm[i]: tmp = layers.batch_norm( input=tmp, act=conv_act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) drop_rate = conv_batchnorm_drop_rate[i] if abs(drop_rate) > 1e-5: tmp = layers.dropout( x=tmp, dropout_prob=drop_rate, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) pool_out = layers.pool2d( input=tmp, pool_size=pool_size, pool_type=pool_type, pool_stride=pool_stride, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) + return pool_out + + +def sequence_conv_pool(input, + num_filters, + filter_size, + act="sigmoid", + pool_type="max", + main_program=None, + startup_program=None): + conv_out = layers.sequence_conv( + input=input, + num_filters=num_filters, + filter_size=filter_size, + act=act, + main_program=main_program, + startup_program=startup_program) + + pool_out = layers.sequence_pool( + input=conv_out, + pool_type=pool_type, + main_program=main_program, + startup_program=startup_program) return pool_out diff --git a/python/paddle/v2/framework/optimizer.py b/python/paddle/v2/framework/optimizer.py index 4c608f96bdf0ca715fc89c0752e891f8c2b80d87..f20865d604f68c8398f299bf8edfd020bfa4e4c5 100644 --- a/python/paddle/v2/framework/optimizer.py +++ b/python/paddle/v2/framework/optimizer.py @@ -1,8 +1,11 @@ from collections import defaultdict import paddle.v2.framework.framework as framework +from paddle.v2.framework.framework import unique_name, Program from paddle.v2.framework.backward import append_backward_ops +from paddle.v2.framework.initializer import ConstantInitializer from paddle.v2.framework.regularizer import append_regularization_ops +from paddle.v2.framework.layer_helper import LayerHelper __all__ = [ 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer', @@ -25,6 +28,7 @@ class Optimizer(object): # to train. These variables are called accumulators. # {accum_name : { paramter_name : accumulator_for_parameter, ...}, ...} self._accumulators = defaultdict(lambda: dict()) + self.helper = None def _append_optimize_op(self, block, param_and_grad): """ append optimize operator to block and return all the added optimize_op @@ -63,7 +67,7 @@ class Optimizer(object): """ pass - def _add_accumulator(self, block, name, param, dtype=None, fill_value=0.0): + def _add_accumulator(self, name, param, dtype=None, fill_value=0.0): """Utility function to add an accumulator for a parameter Args: @@ -77,22 +81,17 @@ class Optimizer(object): param.name in self._accumulators[name]): raise Exception("Accumulator {} already exists for parmeter {}". format(name, param.name)) - global_block = block.program.global_block() - param_shape = list(param.shape) - param_acc = global_block.create_var( - dtype=dtype, shape=param_shape, lod_level=0) - - # Initialize the accumulator with fill_value - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": param_acc}, - attrs={"shape": param_shape, - "value": fill_value}) - - # Add to accumulators dict - self._accumulators[name][param.name] = param_acc + + assert isinstance(self.helper, LayerHelper) + var = self.helper.create_global_variable( + name=unique_name(name), + persistable=True, + dtype=dtype or param.data_type, + type=param.type, + shape=param.shape) + self.helper.set_variable_initializer( + var, initializer=ConstantInitializer(value=float(fill_value))) + self._accumulators[name][param.name] = var def _get_accumulator(self, name, param): """Utility function to fetch an accumulator for a parameter @@ -130,7 +129,10 @@ class Optimizer(object): return increment_op - def create_optimization_pass(self, parameters_and_grads, loss): + def create_optimization_pass(self, + parameters_and_grads, + loss, + startup_program=None): """Add optimization operators to update gradients to variables. Args: @@ -142,6 +144,7 @@ class Optimizer(object): optimization. This will include parameter update ops, global step update ops and any other custom ops required by subclasses to manage their internal state. + :param startup_program: """ # This is a default implementation of create_optimization_pass that # can be shared by most optimizers. This implementation assumes that @@ -151,6 +154,11 @@ class Optimizer(object): # for parameters and extend _finish_update method to add custom ops. # Create any accumulators + program = loss.block.program + self.helper = LayerHelper( + self.__class__.__name__, + main_program=program, + startup_program=startup_program) self._create_accumulators(loss.block, [p[0] for p in parameters_and_grads]) # Create any necessary tensors @@ -177,7 +185,11 @@ class Optimizer(object): return_ops.append(self._increment_global_step(loss.block)) return return_ops - def minimize(self, loss, parameter_list=None, no_grad_set=None): + def minimize(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None): """Add operations to minimize `loss` by updating `parameter_list`. This method combines interface `append_backward_ops()` and @@ -187,7 +199,8 @@ class Optimizer(object): set()) # Add regularization if any params_grads = append_regularization_ops(params_grads) - optimize_ops = self.create_optimization_pass(params_grads, loss) + optimize_ops = self.create_optimization_pass(params_grads, loss, + startup_program) return optimize_ops @@ -202,24 +215,19 @@ class SGDOptimizer(Optimizer): self._learning_rate = learning_rate def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) lr_shape = [1] # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) + self._lr = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=lr_shape, + lod_level=1, + persistable=True) + self.helper.set_variable_initializer( + var=self._lr, initializer=ConstantInitializer(self._learning_rate)) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) - # create the optimize op sgd_op = block.append_op( type=self.type, @@ -255,23 +263,20 @@ class MomentumOptimizer(Optimizer): assert isinstance(block, framework.Block) lr_shape = [1] # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) + self._lr = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=lr_shape, + lod_level=1, + persistable=True) + self.helper.set_variable_initializer( + var=self._lr, initializer=ConstantInitializer(self._learning_rate)) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(block, self._velocity_acc_str, p, 'float32') + self._add_accumulator(self._velocity_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -311,26 +316,22 @@ class AdagradOptimizer(Optimizer): self._epsilon = epsilon def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) lr_shape = [1] # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) + self._lr = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=lr_shape, + lod_level=1, + persistable=True) + self.helper.set_variable_initializer( + var=self._lr, initializer=ConstantInitializer(self._learning_rate)) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(block, self._moment_acc_str, p, 'float32') + self._add_accumulator(self._moment_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -378,51 +379,46 @@ class AdamOptimizer(Optimizer): self._epsilon = epsilon def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) lr_shape = [1] # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) + self._lr = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=lr_shape, + lod_level=1, + persistable=True) + self.helper.set_variable_initializer( + var=self._lr, initializer=ConstantInitializer(self._learning_rate)) def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) - global_block = block.program.global_block() + main_block = block.program.global_block() # Create beta1 and beta2 power tensors beta_shape = [1] - # Create variables for beta1 and beta2 powers - self._beta1_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - self._beta2_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - - # Initialize beta1 and beta2 power accumulators - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta1_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta1}) - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta2_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta2}) + self._beta1_pow_acc = self.helper.create_global_variable( + name=unique_name('beta1_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + self.helper.set_variable_initializer( + self._beta1_pow_acc, initializer=ConstantInitializer(self._beta1)) + + self._beta2_pow_acc = self.helper.create_global_variable( + name=unique_name('beta2_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + + self.helper.set_variable_initializer( + self._beta2_pow_acc, initializer=ConstantInitializer(self._beta2)) # Create accumulator tensors for first and second moments for p in parameters: - self._add_accumulator(block, self._moment1_acc_str, p, 'float32') - self._add_accumulator(block, self._moment2_acc_str, p, 'float32') + self._add_accumulator(self._moment1_acc_str, p) + self._add_accumulator(self._moment2_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -460,14 +456,14 @@ class AdamOptimizer(Optimizer): """Update Beta1 and Beta2 Power accumulators """ assert isinstance(block, framework.Block) - global_block = block.program.global_block() - scale_beta1 = global_block.append_op( + main_block = block.program.global_block() + scale_beta1 = main_block.append_op( type="scale", inputs={"X": self._beta1_pow_acc}, outputs={"Out": self._beta1_pow_acc}, attrs={"scale": self._beta1}) - scale_beta2 = global_block.append_op( + scale_beta2 = main_block.append_op( type="scale", inputs={"X": self._beta2_pow_acc}, outputs={"Out": self._beta2_pow_acc}, @@ -500,43 +496,33 @@ class AdamaxOptimizer(Optimizer): self._epsilon = epsilon def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) lr_shape = [1] # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) + self._lr = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=lr_shape, + lod_level=1, + persistable=True) + self.helper.set_variable_initializer( + var=self._lr, initializer=ConstantInitializer(self._learning_rate)) def _create_accumulators(self, block, parameters): - assert isinstance(block, framework.Block) - - global_block = block.program.global_block() # Create beta1 power accumulator tensor beta_shape = [1] - self._beta1_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - - # Initialize beta1 power accumulator - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta1_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta1}) + self._beta1_pow_acc = self.helper.create_global_variable( + name=unique_name('beta1_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + self.helper.set_variable_initializer( + self._beta1_pow_acc, initializer=ConstantInitializer(self._beta1)) # Create accumulator tensors for first moment and infinity norm for p in parameters: - self._add_accumulator(block, self._moment_acc_str, p, 'float32') - self._add_accumulator(block, self._inf_norm_acc_str, p, 'float32') + self._add_accumulator(self._moment_acc_str, p) + self._add_accumulator(self._inf_norm_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -572,8 +558,8 @@ class AdamaxOptimizer(Optimizer): """Update Beta1 Power accumulator """ assert isinstance(block, framework.Block) - global_block = block.program.global_block() - scale_beta1 = global_block.append_op( + main_block = block.program.global_block() + scale_beta1 = main_block.append_op( type="scale", inputs={"X": self._beta1_pow_acc}, outputs={"Out": self._beta1_pow_acc}, diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 50360e6e729df2957a5c7fe871100b5a53bd9305..2e6710b5fcfe5a531067498e38a4cb93d3165602 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -281,7 +281,8 @@ class OpTest(unittest.TestCase): type(sub_out)) for sub_out_name, expect in sub_out: idx = find_actual(sub_out_name, fetch_list) - actual_t = np.array(outs[idx]) + actual = outs[idx] + actual_t = np.array(actual) expect_t = expect[0] \ if isinstance(expect, tuple) else expect self.assertTrue( @@ -291,11 +292,12 @@ class OpTest(unittest.TestCase): str(place)) if isinstance(expect, tuple): self.assertListEqual( - actual_t.lod(), expect[1], "Output (" + sub_out_name - + ") has different lod at " + str(place)) + actual.lod(), expect[1], "Output (" + sub_out_name + + ") has different lod at " + str(place)) else: idx = find_actual(out_name, fetch_list) - actual_t = outs[idx] + actual = outs[idx] + actual_t = np.array(actual) expect = self.outputs[out_name] expect_t = expect[0] if isinstance(expect, tuple) else expect self.assertTrue( @@ -303,7 +305,7 @@ class OpTest(unittest.TestCase): actual_t, expect_t, atol=atol), "Output (" + out_name + ") has diff at " + str(place)) if isinstance(expect, tuple): - self.assertListEqual(actual_t.lod(), expect[1], + self.assertListEqual(actual.lod(), expect[1], "Output (" + out_name + ") has different lod at " + str(place)) diff --git a/python/paddle/v2/framework/tests/test_accuracy_op.py b/python/paddle/v2/framework/tests/test_accuracy_op.py index 02be9a02910bee3eae63e12cceaa51cf53591539..6536c297e8e559bf04fe6ef3b0e2dadd1914eb87 100644 --- a/python/paddle/v2/framework/tests/test_accuracy_op.py +++ b/python/paddle/v2/framework/tests/test_accuracy_op.py @@ -7,12 +7,13 @@ class TestAccuracyOp(OpTest): def setUp(self): self.op_type = "accuracy" n = 8192 - infer = np.random.randint(0, 2, (n, 1)).astype("int") - label = np.random.randint(0, 2, (n, )).astype("int") - self.inputs = {'Inference': infer, "Label": label} + infer = np.random.random((n, 1)).astype("float32") + indices = np.random.randint(0, 2, (n, 1)) + label = np.random.randint(0, 2, (n, 1)) + self.inputs = {'Out': infer, 'Indices': indices, "Label": label} num_correct = 0 for rowid in xrange(n): - for ele in infer[rowid]: + for ele in indices[rowid]: if ele == label[rowid]: num_correct += 1 break diff --git a/python/paddle/v2/framework/tests/test_auc_op.py b/python/paddle/v2/framework/tests/test_auc_op.py index 65f679cfccccae41b8924bc68833c1703dd3671d..26ea905d88093605dff820b178996a5724becf82 100644 --- a/python/paddle/v2/framework/tests/test_auc_op.py +++ b/python/paddle/v2/framework/tests/test_auc_op.py @@ -6,10 +6,11 @@ from op_test import OpTest class TestAucOp(OpTest): def setUp(self): self.op_type = "auc" - pred = np.random.random((128)).astype("float32") - labels = np.random.randint(0, 2, (128, )) + pred = np.random.random((128, 2)).astype("float32") + indices = np.random.randint(0, 2, (128, 2)) + labels = np.random.randint(0, 2, (128, 1)) num_thresholds = 200 - self.inputs = {'Inference': pred, 'Label': labels} + self.inputs = {'Out': pred, 'Indices': indices, 'Label': labels} self.attrs = {'curve': 'ROC', 'num_thresholds': num_thresholds} # NOTE: sklearn use a different way to generate thresholds # which will cause the result differs slightly: @@ -31,12 +32,12 @@ class TestAucOp(OpTest): tp, fn, tn, fp = 0, 0, 0, 0 for i, lbl in enumerate(labels): if lbl: - if pred[i] >= thresh: + if pred[i, 0] >= thresh: tp += 1 else: fn += 1 else: - if pred[i] >= thresh: + if pred[i, 0] >= thresh: fp += 1 else: tn += 1 @@ -62,6 +63,5 @@ class TestAucOp(OpTest): self.check_output() -# TODO(typhoonzero): add this back till we fix it -#if __name__ == "__main__": -# unittest.main() +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv2dtranspose_op.py b/python/paddle/v2/framework/tests/test_conv2d_transpose_op.py similarity index 90% rename from python/paddle/v2/framework/tests/test_conv2dtranspose_op.py rename to python/paddle/v2/framework/tests/test_conv2d_transpose_op.py index 53604c58b70a534dff6b0a668d380fb8f10f53f6..999a0bdc629010d96a8db31b317ba7a65bf35773 100644 --- a/python/paddle/v2/framework/tests/test_conv2dtranspose_op.py +++ b/python/paddle/v2/framework/tests/test_conv2d_transpose_op.py @@ -45,23 +45,36 @@ class TestConv2dTransposeOp(OpTest): filter_ = np.random.random(self.filter_size).astype("float32") output = conv2dtranspose_forward_naive( input_, filter_, conv2dtranspose_param).astype('float32') - # print 'deconv output py', output, output.shape self.inputs = {'Input': input_, 'Filter': filter_} self.attrs = { 'strides': self.stride, 'paddings': self.pad, - # 'dilations': self.dilations + 'dilations': self.dilations } self.outputs = {'Output': output} def test_check_output(self): - print 'check output here' + print 'check output here for', self.op_type self.check_output() - def test_check_grad(self): + def init_test_case(self): + self.pad = [0, 0] + self.stride = [1, 1] + self.dilations = [1, 1] + self.input_size = [2, 3, 5, 5] # NCHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3] + + def init_op_type(self): + self.op_type = "conv2d_transpose" + + def test_check_grad_no_input(self): self.check_grad( - set(['Input', 'Filter']), 'Output', max_relative_error=0.05) + ['Filter'], + 'Output', + max_relative_error=0.05, + no_grad_set=set(['Input'])) def test_check_grad_no_filter(self): self.check_grad( @@ -70,33 +83,15 @@ class TestConv2dTransposeOp(OpTest): max_relative_error=0.05, no_grad_set=set(['Filter'])) - def test_check_grad_no_input(self): + def test_check_grad(self): self.check_grad( - ['Filter'], - 'Output', - max_relative_error=0.05, - no_grad_set=set(['Input'])) + set(['Input', 'Filter']), 'Output', max_relative_error=0.05) - def init_test_case(self): - self.pad = [0, 0] - self.stride = [1, 1] - self.dilations = [1, 1] - self.input_size = [2, 3, 5, 5] # NCHW - f_c = self.input_size[1] - self.filter_size = [f_c, 6, 3, 3] +class TestCudnn(TestConv2dTransposeOp): def init_op_type(self): - self.op_type = "conv2dtranspose" - + self.op_type = "conv2d_transpose_cudnn" -""" -class TestCudnn(TestConv2dOp): - def init_group(self): - self.groups = 1 - - def init_op_type(self): - self.op_type = "conv_cudnn" -""" if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_crf_decoding_op.py b/python/paddle/v2/framework/tests/test_crf_decoding_op.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2b996bf430d5a0edaa0de459a937adffd9f8f6 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_crf_decoding_op.py @@ -0,0 +1,146 @@ +import unittest +import random +import numpy as np + +from op_test import OpTest + + +class CRFDecoding(object): + def __init__(self, emission_weights, transition_weights, + seq_start_positions): + assert (emission_weights.shape[0] == seq_start_positions[-1]) + self.tag_num = emission_weights.shape[1] + self.seq_num = len(seq_start_positions) - 1 + + self.seq_start_positions = seq_start_positions + self.x = emission_weights + + self.a = transition_weights[0, :] + self.b = transition_weights[1, :] + self.w = transition_weights[2:, :] + + self.track = np.zeros( + (seq_start_positions[-1], self.tag_num), dtype="int32") + self.decoded_path = np.zeros( + (seq_start_positions[-1], 1), dtype="int32") + + def _decode_one_sequence(self, decoded_path, x): + seq_len, tag_num = x.shape + alpha = np.zeros((seq_len, tag_num), dtype="float64") + track = np.zeros((seq_len, tag_num), dtype="int32") + + for i in range(tag_num): + alpha[0, i] = self.a[i] + x[0, i] + + for k in range(1, seq_len): + for i in range(tag_num): + max_score = -np.finfo("float64").max + max_idx = 0 + for j in range(tag_num): + score = alpha[k - 1, j] + self.w[j, i] + if score > max_score: + max_score = score + max_idx = j + alpha[k, i] = max_score + x[k, i] + track[k, i] = max_idx + + max_score = -np.finfo("float64").max + max_idx = 0 + for i in range(tag_num): + score = alpha[seq_len - 1, i] + self.b[i] + if score > max_score: + max_score = score + max_idx = i + + decoded_path[-1] = max_idx + for i in range(seq_len - 1, 0, -1): + decoded_path[i - 1] = max_idx = track[i, max_idx] + + def decode(self): + for i in range(self.seq_num): + start = self.seq_start_positions[i] + end = self.seq_start_positions[i + 1] + self._decode_one_sequence(self.decoded_path[start:end, :], + self.x[start:end, :]) + return self.decoded_path + + +class TestCRFDecodingOp1(OpTest): + """ + Compare the dynamic program with random generated parameters and inputs + with grouth truth not being given. + """ + + def set_test_data(self): + SEQ_NUM = 3 + TAG_NUM = 17 + MAX_SEQ_LEN = 10 + + lod = [[0]] + for i in range(SEQ_NUM): + lod[-1].append(lod[-1][-1] + random.randint(1, MAX_SEQ_LEN)) + emission = np.random.uniform(-1, 1, + [lod[-1][-1], TAG_NUM]).astype("float64") + transition = np.random.uniform(-0.5, 0.5, + [TAG_NUM + 2, TAG_NUM]).astype("float64") + + self.inputs = { + "Emission": (emission, lod), + "Transition": transition, + } + + decoder = CRFDecoding(emission, transition, lod[0]) + decoded_path = decoder.decode() + + self.outputs = {"ViterbiPath": decoded_path} + + def setUp(self): + self.op_type = "crf_decoding" + self.set_test_data() + + def test_check_output(self): + self.check_output() + + +class TestCRFDecodingOp2(OpTest): + """ + Compare the dynamic program with brute force computation with + ground truth being given. + """ + + def setUp(self): + self.op_type = "crf_decoding" + TAG_NUM = 5 + + lod = [[0, 1, 3, 6, 10]] + transition = np.repeat( + np.arange( + TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0) + emission = np.repeat( + np.arange( + TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + lod[-1][-1], + axis=0) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(lod[-1][-1], 1), dtype="int32") + predicted_labels = np.ones( + (lod[-1][-1], 1), dtype="int32") * (TAG_NUM - 1) + expected_output = (labels == predicted_labels).astype("int32") + + self.inputs = { + "Emission": (emission, lod), + "Transition": transition, + "Label": (labels, lod) + } + + self.outputs = {"ViterbiPath": expected_output} + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_evaluator.py b/python/paddle/v2/framework/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..37dbfbc06bcd0da7e11924a048679c74a1cfb373 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_evaluator.py @@ -0,0 +1,64 @@ +from paddle.v2.framework.evaluator import Evaluator +from paddle.v2.framework.op import Operator +import paddle.v2.framework.core as core +import unittest +import op_test +import numpy as np + + +class TestEvaluator(unittest.TestCase): + def setup(self, scope, inputs, outputs): + def __create_var__(var_name, arr): + np_arr = np.array(arr) + scope.var(var_name) + # tensor = var.get_tensor() + # tensor.set_dims(np_arr.shape) + + for var_name, arr in inputs.iteritems(): + __create_var__(var_name, arr) + + for var_name, arr in outputs.iteritems(): + __create_var__(var_name, arr) + + def test_evaluator(self): + + inputs = { + 'Inference': np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 1]]).T, + 'Label': np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) + } + outputs = {'Accuracy': np.array([0.9])} + out_name = 'Accuracy' + + places = [core.CPUPlace()] + if core.is_compile_gpu(): + places.append(core.GPUPlace(0)) + + for place in places: + scope = core.Scope() + self.setup(scope, inputs, outputs) + + evaluator = Evaluator( + scope, + operator='accuracy', + input='Inference', + label='Label', + output=out_name, + place=place) + op_test.set_input(scope, evaluator.op, inputs, place) + ctx = core.DeviceContext.create(place) + + for i in range(10): # simulate 10 mini-batches + evaluator.evaluate(ctx) + + actual = np.array(scope.find_var(out_name).get_tensor()) + print actual + + self.assertTrue( + np.allclose( + actual, outputs[out_name], atol=1e-5), + "output name: " + out_name + " has diff.") + + +if __name__ == '__main__': + exit(0) + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_executor_and_mul.py b/python/paddle/v2/framework/tests/test_executor_and_mul.py index 35f775711167ce0d210044ab4cb382db802f39a5..c885cfbebd4b665ddf50adbc43673942dc949a0b 100644 --- a/python/paddle/v2/framework/tests/test_executor_and_mul.py +++ b/python/paddle/v2/framework/tests/test_executor_and_mul.py @@ -2,7 +2,7 @@ import unittest from paddle.v2.framework.layers import mul, data import paddle.v2.framework.core as core from paddle.v2.framework.executor import Executor -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program import numpy @@ -23,7 +23,7 @@ class TestExecutor(unittest.TestCase): tensor_b = core.LoDTensor() tensor_b.set(b_np, place) exe = Executor(place) - outs = exe.run(g_program, + outs = exe.run(g_main_program, feed={'a': tensor_a, 'b': tensor_b}, fetch_list=[out]) diff --git a/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py b/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py index 065a9133dca25fac988f9493c1527e0d8f9821dc..319ae52fb38ee122808a6e6b0e44d3a3787d8e53 100644 --- a/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py @@ -3,13 +3,27 @@ import numpy as np from op_test import OpTest -class TestFillConstantBatchSizeLikeOp(OpTest): +class TestFillConstantBatchSizeLikeWhenFirstDimIsBatchSize(OpTest): def setUp(self): self.op_type = "fill_constant_batch_size_like" self.inputs = {'Input': np.random.random((219, 232)).astype("float32")} - self.attrs = {'value': 3.5, 'shape': [-1, 132, 777]} + self.attrs = {'value': 3.5, 'shape': [-1, 132, 7]} - out = np.random.random((219, 132, 777)).astype("float32") + out = np.random.random((219, 132, 7)).astype("float32") + out.fill(3.5) + self.outputs = {'Out': out} + + def test_check_output(self): + self.check_output() + + +class TestFillConstantBatchSizeLikeWhenSecondDimIsBatchSize(OpTest): + def setUp(self): + self.op_type = "fill_constant_batch_size_like" + self.inputs = {'Input': np.random.random((219, 232)).astype("float32")} + self.attrs = {'value': 3.5, 'shape': [132, -1, 7], 'dim_idx': 1} + + out = np.random.random((132, 232, 7)).astype("float32") out.fill(3.5) self.outputs = {'Out': out} diff --git a/python/paddle/v2/framework/tests/test_fit_a_line.py b/python/paddle/v2/framework/tests/test_fit_a_line.py index 7c2ef61fe103655369fd6fe68733e810d4e19d1d..174ee74c3bc8952d296a2ffa205108f2a8e12cfb 100644 --- a/python/paddle/v2/framework/tests/test_fit_a_line.py +++ b/python/paddle/v2/framework/tests/test_fit_a_line.py @@ -3,40 +3,44 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program, g_main_program from paddle.v2.framework.io import save_persistables, load_persistables from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() x = layers.data( name='x', shape=[13], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) y_predict = layers.fc(input=x, size=1, act=None, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) y = layers.data( name='y', shape=[1], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.square_error_cost( - input=y_predict, label=y, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) + input=y_predict, + label=y, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +opts = sgd_optimizer.minimize(avg_cost, startup_program) BATCH_SIZE = 20 @@ -48,12 +52,12 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): - save_persistables(exe, "./fit_a_line.model/", program=program) - load_persistables(exe, "./fit_a_line.model/", program=program) + save_persistables(exe, "./fit_a_line.model/", main_program=main_program) + load_persistables(exe, "./fit_a_line.model/", main_program=main_program) for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("float32") @@ -65,7 +69,7 @@ for pass_id in range(PASS_NUM): tensor_y = core.LoDTensor() tensor_y.set(y_data, place) # print tensor_y.get_dims() - outs = exe.run(program, + outs = exe.run(main_program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost]) diff --git a/python/paddle/v2/framework/tests/test_gaussian_random_op.py b/python/paddle/v2/framework/tests/test_gaussian_random_op.py index 8b7779667d5e806c06b333527f774c7987ce7e73..0dc7e091a5c8dd046f36cab7f79a15b2281cdd90 100644 --- a/python/paddle/v2/framework/tests/test_gaussian_random_op.py +++ b/python/paddle/v2/framework/tests/test_gaussian_random_op.py @@ -19,7 +19,7 @@ class TestGaussianRandomOp(unittest.TestCase): op = Operator( "gaussian_random", Out='Out', - dims=[1000, 784], + shape=[1000, 784], mean=.0, std=1., seed=10) diff --git a/python/paddle/v2/framework/tests/test_image_classification_layer.py b/python/paddle/v2/framework/tests/test_image_classification_layer.py index 908cf44b88a5de88690f5e17a1da1b5f8b1d8079..b1a267ec32b1c937b946bee82e41b846ebbf1288 100644 --- a/python/paddle/v2/framework/tests/test_image_classification_layer.py +++ b/python/paddle/v2/framework/tests/test_image_classification_layer.py @@ -9,8 +9,8 @@ def conv_block(input, num_filter, groups, dropouts, - program=None, - init_program=None): + main_program=None, + startup_program=None): return nets.img_conv_group( input=input, pool_size=2, @@ -21,54 +21,81 @@ def conv_block(input, conv_with_batchnorm=True, conv_batchnorm_drop_rate=dropouts, pool_type='max', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) class TestLayer(unittest.TestCase): def test_batch_norm_layer(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program) + main_program=main_program) layers.batch_norm( - input=images, program=program, init_program=init_program) + input=images, + main_program=main_program, + startup_program=startup_program) - #print str(program) + # print str(main_program) def test_dropout_layer(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program) + main_program=main_program) layers.dropout( x=images, dropout_prob=0.5, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) - #print str(program) + # print str(main_program) def test_img_conv_group(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program, - init_program=init_program) - conv1 = conv_block(images, 64, 2, [0.3, 0], program, init_program) - conv2 = conv_block(conv1, 256, 3, [0.4, 0.4, 0], program, init_program) + main_program=main_program, + startup_program=startup_program) + conv1 = conv_block(images, 64, 2, [0.3, 0], main_program, + startup_program) + conv2 = conv_block(conv1, 256, 3, [0.4, 0.4, 0], main_program, + startup_program) - # print str(program) + # print str(main_program) + + def test_elementwise_add_with_act(self): + main_program = Program() + startup_program = Program() + image1 = layers.data( + name='pixel1', + shape=[3, 48, 48], + data_type='float32', + main_program=main_program, + startup_program=startup_program) + image2 = layers.data( + name='pixel2', + shape=[3, 48, 48], + data_type='float32', + main_program=main_program, + startup_program=startup_program) + out = layers.elementwise_add( + x=image1, + y=image2, + act='relu', + main_program=main_program, + startup_program=startup_program) + # print(main_program) if __name__ == '__main__': diff --git a/python/paddle/v2/framework/tests/test_image_classification_train.py b/python/paddle/v2/framework/tests/test_image_classification_train.py index 4eb9051261ee6786ba78f62ea3bfd89ae90e1d74..a4165da9703c55ae3347123409407f0cae30856f 100644 --- a/python/paddle/v2/framework/tests/test_image_classification_train.py +++ b/python/paddle/v2/framework/tests/test_image_classification_train.py @@ -1,22 +1,139 @@ +import numpy as np import paddle.v2 as paddle +import paddle.v2.framework.core as core import paddle.v2.framework.layers as layers import paddle.v2.framework.nets as nets -import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer - -from paddle.v2.framework.framework import Program, g_program from paddle.v2.framework.executor import Executor +from paddle.v2.framework.framework import g_startup_program, g_main_program +from paddle.v2.framework.initializer import XavierInitializer -import numpy as np +def resnet_cifar10(input, depth=32, main_program=None, startup_program=None): + def conv_bn_layer(input, + ch_out, + filter_size, + stride, + padding, + act='relu', + main_program=None, + startup_program=None): + tmp = layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=False, + main_program=main_program, + startup_program=startup_program) + return layers.batch_norm( + input=tmp, + act=act, + main_program=main_program, + startup_program=startup_program) + + def shortcut(input, ch_in, ch_out, stride, program, init_program): + if ch_in != ch_out: + return conv_bn_layer(input, ch_out, 1, stride, 0, None, program, + init_program) + else: + return input + + def basicblock(input, + ch_in, + ch_out, + stride, + main_program=main_program, + startup_program=startup_program): + tmp = conv_bn_layer( + input, + ch_out, + 3, + stride, + 1, + main_program=main_program, + startup_program=startup_program) + tmp = conv_bn_layer( + tmp, + ch_out, + 3, + 1, + 1, + act=None, + main_program=main_program, + startup_program=startup_program) + short = shortcut(input, ch_in, ch_out, stride, main_program, + startup_program) + return layers.elementwise_add( + x=tmp, + y=short, + act='relu', + main_program=main_program, + startup_program=startup_program) -def vgg16_bn_drop(input, program, init_program): + def layer_warp(block_func, input, ch_in, ch_out, count, stride, program, + startup_program): + tmp = block_func(input, ch_in, ch_out, stride, program, startup_program) + for i in range(1, count): + tmp = block_func(tmp, ch_out, ch_out, 1, program, startup_program) + return tmp + + assert (depth - 2) % 6 == 0 + n = (depth - 2) / 6 + conv1 = conv_bn_layer( + input=input, + ch_out=16, + filter_size=3, + stride=1, + padding=1, + main_program=main_program, + startup_program=startup_program) + res1 = layer_warp( + basicblock, + conv1, + 16, + 16, + n, + 1, + main_program=main_program, + startup_program=startup_program) + res2 = layer_warp( + basicblock, + res1, + 16, + 32, + n, + 2, + main_program=main_program, + startup_program=startup_program) + res3 = layer_warp( + basicblock, + res2, + 32, + 64, + n, + 2, + main_program=main_program, + startup_program=startup_program) + pool = layers.pool2d( + input=res3, + pool_size=8, + pool_type='avg', + pool_stride=1, + main_program=main_program, + startup_program=startup_program) + return pool + + +def vgg16_bn_drop(input, main_program=None, startup_program=None): def conv_block(input, num_filter, groups, dropouts, - program=None, - init_program=None): + main_program=None, + startup_program=None): return nets.img_conv_group( input=input, pool_size=2, @@ -27,66 +144,75 @@ def vgg16_bn_drop(input, program, init_program): conv_with_batchnorm=True, conv_batchnorm_drop_rate=dropouts, pool_type='max', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) - conv1 = conv_block(input, 64, 2, [0.3, 0], program, init_program) - conv2 = conv_block(conv1, 128, 2, [0.4, 0], program, init_program) - conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0], program, init_program) - conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0], program, init_program) - conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0], program, init_program) + conv1 = conv_block(input, 64, 2, [0.3, 0], main_program, startup_program) + conv2 = conv_block(conv1, 128, 2, [0.4, 0], main_program, startup_program) + conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0], main_program, + startup_program) + conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0], main_program, + startup_program) + conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0], main_program, + startup_program) drop = layers.dropout( - x=conv5, dropout_prob=0.5, program=program, init_program=init_program) + x=conv5, + dropout_prob=0.5, + main_program=main_program, + startup_program=startup_program) fc1 = layers.fc(input=drop, size=512, act=None, - program=program, - init_program=init_program) + param_attr={"initializer": XavierInitializer()}, + main_program=main_program, + startup_program=startup_program) reshape1 = layers.reshape( x=fc1, shape=list(fc1.shape + (1, 1)), - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) bn = layers.batch_norm( - input=reshape1, act='relu', program=program, init_program=init_program) + input=reshape1, + act='relu', + main_program=main_program, + startup_program=startup_program) drop2 = layers.dropout( - x=bn, dropout_prob=0.5, program=program, init_program=init_program) + x=bn, + dropout_prob=0.5, + main_program=main_program, + startup_program=startup_program) fc2 = layers.fc(input=drop2, size=512, act=None, - program=program, - init_program=init_program) + param_attr={"initializer": XavierInitializer()}, + main_program=main_program, + startup_program=startup_program) return fc2 -init_program = Program() -program = Program() - classdim = 10 data_shape = [3, 32, 32] -images = layers.data( - name='pixel', shape=data_shape, data_type='float32', program=program) - -label = layers.data( - name='label', - shape=[1], - data_type='int64', - program=program, - init_program=init_program) -vgg_net = vgg16_bn_drop(images, program, init_program) -predict = layers.fc(input=vgg_net, - size=classdim, - act='softmax', - program=program, - init_program=init_program) -cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) - -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +images = layers.data(name='pixel', shape=data_shape, data_type='float32') +label = layers.data(name='label', shape=[1], data_type='int64') + +# Add neural network config +# option 1. resnet +# net = resnet_cifar10(images, 32) +# option 2. vgg +net = vgg16_bn_drop(images) + +# print(program) + +predict = layers.fc(input=net, size=classdim, act='softmax') +cost = layers.cross_entropy(input=predict, label=label) +avg_cost = layers.mean(x=cost) +accuracy = layers.accuracy(input=predict, label=label) + +# optimizer = optimizer.SGDOptimizer(learning_rate=0.001) +optimizer = optimizer.AdamOptimizer(learning_rate=0.001) +opts = optimizer.minimize(avg_cost) BATCH_SIZE = 128 PASS_NUM = 1 @@ -99,7 +225,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(g_startup_program, feed={}, fetch_list=[]) for pass_id in range(PASS_NUM): batch_id = 0 @@ -117,14 +243,15 @@ for pass_id in range(PASS_NUM): tensor_img.set(img_data, place) tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(g_main_program, feed={"pixel": tensor_img, "label": tensor_y}, - fetch_list=[avg_cost]) + fetch_list=[avg_cost, accuracy]) loss = np.array(outs[0]) - # print("pass_id:" + str(pass_id) + " batch_id:" + str(batch_id) + - # " loss:" + str(loss)) + acc = np.array(outs[1]) + print("pass_id:" + str(pass_id) + " batch_id:" + str(batch_id) + + " loss:" + str(loss) + " acc:" + str(acc)) batch_id = batch_id + 1 if batch_id > 1: diff --git a/python/paddle/v2/framework/tests/test_inference_model_io.py b/python/paddle/v2/framework/tests/test_inference_model_io.py index 4487ab989f3c5da92e086c1fd395c3d776dce9a9..d273387a35820a27f87ef4569f158db4fe670864 100644 --- a/python/paddle/v2/framework/tests/test_inference_model_io.py +++ b/python/paddle/v2/framework/tests/test_inference_model_io.py @@ -3,7 +3,7 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program, g_main_program from paddle.v2.framework.io import save_inference_model, load_inference_model import paddle.v2.framework.executor as executor import unittest @@ -20,31 +20,31 @@ class TestBook(unittest.TestCase): name='x', shape=[2], data_type='float32', - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) y = layers.data( name='y', shape=[1], data_type='float32', - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) y_predict = layers.fc(input=x, size=1, act=None, - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) cost = layers.square_error_cost( input=y_predict, label=y, - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) avg_cost = layers.mean( - x=cost, program=program, init_program=init_program) + x=cost, main_program=program, startup_program=init_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) - opts = sgd_optimizer.minimize(avg_cost) + opts = sgd_optimizer.minimize(avg_cost, init_program) place = core.CPUPlace() exe = executor.Executor(place) diff --git a/python/paddle/v2/framework/tests/test_initializer.py b/python/paddle/v2/framework/tests/test_initializer.py new file mode 100644 index 0000000000000000000000000000000000000000..bd4d2e39d770aebb7468d516f463533185ea8680 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_initializer.py @@ -0,0 +1,227 @@ +import numpy as np +import unittest + +import paddle.v2.framework.framework as framework +import paddle.v2.framework.initializer as initializer + +DELTA = 0.00001 + + +class TestConstantInitializer(unittest.TestCase): + def test_constant_initializer_default_value(self): + """Test the constant initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'fill_constant') + self.assertAlmostEqual(init_op.attr('value'), 0.0, delta=DELTA) + + def test_constant_initializer(self): + """Test constant initializer with supplied value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer(2.3)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'fill_constant') + self.assertAlmostEqual(init_op.attr('value'), 2.3, delta=DELTA) + + +class TestUniformInitializer(unittest.TestCase): + def test_uniform_initializer_default_value(self): + """Test the uniform initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), -1.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_uniform_initializer(self): + """Test uniform initializer with supplied attributes + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, 3.1, 123)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), -4.2, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), 3.1, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 123) + + +class TestNormalInitializer(unittest.TestCase): + def test_normal_initializer_default_value(self): + """Test the normal initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_initializer(self): + """Test normal initializer with supplied attributes + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(2.3, 1.9, 123)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 2.3, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.9, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 123) + + +class TestXavierInitializer(unittest.TestCase): + def test_uniform_xavier_initializer(self): + """Test Xavier initializer with uniform distribution on + for matrix multiply. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_uniform_xavier_initializer_conv(self): + """Test Xavier initializer with uniform distribution on + for convolutions. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + receptive_field_size = float(15 * 20) + limit = np.sqrt(6.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_xavier_initializer(self): + """Test Xavier initializer with normal distribution on + for matrix multiply. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + std = np.sqrt(2.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_xavier_initializer_conv(self): + """Test Xavier initializer with normal distribution on + for convolutions. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + receptive_field_size = float(15 * 20) + std = np.sqrt(2.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_xavier_initializer_supplied_arguments(self): + """Test the Xavier initializer with supplied arguments + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer( + fan_in=12, fan_out=23, seed=134)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (12 + 23)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 134) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_layers.py b/python/paddle/v2/framework/tests/test_layers.py index 5cbe790e3f019f5dcf6b201c4744e7502141ed99..716963fb431a8aca70e1b87efaf268595af2b09e 100644 --- a/python/paddle/v2/framework/tests/test_layers.py +++ b/python/paddle/v2/framework/tests/test_layers.py @@ -1,6 +1,6 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.nets as nets -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program, g_main_program import paddle.v2.framework.core as core import unittest @@ -9,15 +9,15 @@ class TestBook(unittest.TestCase): def test_fit_a_line(self): program = Program() x = layers.data( - name='x', shape=[13], data_type='float32', program=program) - y_predict = layers.fc(input=x, size=1, act=None, program=program) + name='x', shape=[13], data_type='float32', main_program=program) + y_predict = layers.fc(input=x, size=1, act=None, main_program=program) y = layers.data( - name='y', shape=[1], data_type='float32', program=program) + name='y', shape=[1], data_type='float32', main_program=program) cost = layers.square_error_cost( - input=y_predict, label=y, program=program) + input=y_predict, label=y, main_program=program) - avg_cost = layers.mean(x=cost, program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) program.append_backward(avg_cost) print str(program) @@ -27,26 +27,42 @@ class TestBook(unittest.TestCase): # Change g_program, so the rest layers use `g_program` images = layers.data( - name='pixel', shape=[784], data_type='float32', program=program) + name='pixel', + shape=[784], + data_type='float32', + main_program=program) label = layers.data( - name='label', shape=[1], data_type='int32', program=program) - hidden1 = layers.fc(input=images, size=128, act='relu', program=program) - hidden2 = layers.fc(input=hidden1, size=64, act='relu', program=program) + name='label', shape=[1], data_type='int32', main_program=program) + hidden1 = layers.fc(input=images, + size=128, + act='relu', + main_program=program) + hidden2 = layers.fc(input=hidden1, + size=64, + act='relu', + main_program=program) predict = layers.fc(input=hidden2, size=10, act='softmax', - program=program) - cost = layers.cross_entropy(input=predict, label=label, program=program) - avg_cost = layers.mean(x=cost, program=program) + main_program=program) + cost = layers.cross_entropy( + input=predict, label=label, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) print str(program) def test_simple_conv2d(self): program = Program() images = layers.data( - name='pixel', shape=[3, 48, 48], data_type='int32', program=program) + name='pixel', + shape=[3, 48, 48], + data_type='int32', + main_program=program) layers.conv2d( - input=images, num_filters=3, filter_size=[4, 4], program=program) + input=images, + num_filters=3, + filter_size=[4, 4], + main_program=program) print str(program) @@ -57,9 +73,9 @@ class TestBook(unittest.TestCase): name='pixel', shape=[1, 28, 28], data_type='float32', - program=program) + main_program=program) label = layers.data( - name='label', shape=[1], data_type='int32', program=program) + name='label', shape=[1], data_type='int32', main_program=program) conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, @@ -67,7 +83,7 @@ class TestBook(unittest.TestCase): pool_size=2, pool_stride=2, act="relu", - program=program) + main_program=program) conv_pool_2 = nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -75,14 +91,15 @@ class TestBook(unittest.TestCase): pool_size=2, pool_stride=2, act="relu", - program=program) + main_program=program) predict = layers.fc(input=conv_pool_2, size=10, act="softmax", - program=program) - cost = layers.cross_entropy(input=predict, label=label, program=program) - avg_cost = layers.mean(x=cost, program=program) + main_program=program) + cost = layers.cross_entropy( + input=predict, label=label, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) program.append_backward(avg_cost) @@ -93,58 +110,58 @@ class TestBook(unittest.TestCase): dict_size = 10000 embed_size = 32 first_word = layers.data( - name='firstw', shape=[1], data_type='int64', program=program) + name='firstw', shape=[1], data_type='int64', main_program=program) second_word = layers.data( - name='secondw', shape=[1], data_type='int64', program=program) + name='secondw', shape=[1], data_type='int64', main_program=program) third_word = layers.data( - name='thirdw', shape=[1], data_type='int64', program=program) + name='thirdw', shape=[1], data_type='int64', main_program=program) forth_word = layers.data( - name='forthw', shape=[1], data_type='int64', program=program) + name='forthw', shape=[1], data_type='int64', main_program=program) next_word = layers.data( - name='nextw', shape=[1], data_type='int64', program=program) + name='nextw', shape=[1], data_type='int64', main_program=program) embed_first = layers.embedding( input=first_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_second = layers.embedding( input=second_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_third = layers.embedding( input=third_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_forth = layers.embedding( input=forth_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1, - program=program) + main_program=program) hidden1 = layers.fc(input=concat_embed, size=256, act='sigmoid', - program=program) + main_program=program) predict_word = layers.fc(input=hidden1, size=dict_size, act='softmax', - program=program) + main_program=program) cost = layers.cross_entropy( - input=predict_word, label=next_word, program=program) - avg_cost = layers.mean(x=cost, program=program) + input=predict_word, label=next_word, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) print str(program) diff --git a/python/paddle/v2/framework/tests/test_linear_chain_crf_op.py b/python/paddle/v2/framework/tests/test_linear_chain_crf_op.py new file mode 100644 index 0000000000000000000000000000000000000000..6f06a66c825b37ee91214efc0a29a58f0b9057f9 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_linear_chain_crf_op.py @@ -0,0 +1,142 @@ +import unittest +import random +import numpy as np + +from op_test import OpTest + + +class LinearChainCrfForward(object): + def __init__(self, seq_start_positions, emission_weights, emission_row_max, + emission_exps, transition_weights, transition_exps, labels): + self.tag_num = emission_weights.shape[1] + self.seq_num = len(seq_start_positions) - 1 + + self.seq_start_positions = seq_start_positions + self.labels = labels + self.x = emission_weights + + self.x_row_max = emission_row_max + self.x_exps = emission_exps + + # unnormalized logits of the transition weights for the start mark. + self.a = transition_weights[0, :] + self.a_exps = transition_exps[0, :] + # unnormalized logits of the transition weights for the end mark. + self.b = transition_weights[1, :] + self.b_exps = transition_exps[1, :] + # unnormalized logits of the transition weights for all the other tags. + self.w = transition_weights[2:, :] + self.w_exps = transition_exps[2:, :] + + # The output of linear chain crf operator. + # alpha is a memo table in dynamic programming to caculate + # nomalization factor. + self.alpha = np.zeros( + (seq_start_positions[-1], self.tag_num), dtype="float64") + self.log_likelihood = np.zeros((self.seq_num, 1)) + + def _l1_norm(self, x): + s = np.sum(x) + x /= s + return s + + def _forward_a_sequence(self, x, x_row_max, x_exps, label, alpha): + seq_len = x_row_max.shape[0] + log_likelihood = 0. + + for i in range(self.tag_num): + alpha[0, i] = self.a_exps[i] * x_exps[0, i] + log_likelihood = -x_row_max[0] - np.log(self._l1_norm(alpha[0, :])) + + # calculate the unnormalized logits of the normalization factor. + for k in range(1, seq_len): + for i in range(self.tag_num): + s = 0. + for j in range(self.tag_num): + s += alpha[k - 1, j] * self.w_exps[j, i] + alpha[k, i] = x_exps[k, i] * s + log_likelihood -= x_row_max[k] + np.log(self._l1_norm(alpha[k, :])) + s = 0. + for i in range(self.tag_num): + s += alpha[-1, i] * self.b_exps[i] + log_likelihood -= np.log(s) + + # calculate the nominator part. + log_likelihood += ( + self.a[label[0]] + x[0, label[0]] + self.b[label[-1]]) + + for k in range(1, seq_len): + log_likelihood += (x[k, label[k]] + self.w[label[k - 1], label[k]]) + return -log_likelihood + + def crf_forward_compute(self): + for i in range(self.seq_num): + start = self.seq_start_positions[i] + end = self.seq_start_positions[i + 1] + + self.log_likelihood[i] = self._forward_a_sequence( + self.x[start:end, :], self.x_row_max[start:end, :], + self.x_exps[start:end, :], self.labels[start:end, :], + self.alpha[start:end, :]) + return self.alpha, self.log_likelihood + + +class TestLinearChainCrfOp(OpTest): + def set_test_data(self): + # TODO(caoying) Fix the unittest by: add the boundary cases when + # sequence lengths are 1, 2, and 3. + + SEQ_NUM = 3 + TAG_NUM = 17 + MAX_SEQ_LEN = 5 + + # the linear_chain_crf operator only supports sequence (LoD level = 1) + lod = [[0]] + for i in range(SEQ_NUM): + lod[-1].append(lod[-1][-1] + random.randint(1, MAX_SEQ_LEN)) + emission = np.random.uniform(-1, 1, + [lod[-1][-1], TAG_NUM]).astype("float64") + emission_row_max = np.amax(emission, axis=1, keepdims=True) + emission_exps = np.exp(emission - emission_row_max) + + transition = np.random.uniform(-0.5, 0.5, + [TAG_NUM + 2, TAG_NUM]).astype("float64") + transition_exps = np.exp(transition) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(lod[-1][-1], 1), dtype="int32") + + self.inputs = { + "Emission": (emission, lod), + "Transition": transition, + "Label": (labels, lod) + } + crf = LinearChainCrfForward(lod[0], emission, emission_row_max, + emission_exps, transition, transition_exps, + labels) + alpha, log_likelihood = crf.crf_forward_compute() + + self.outputs = { + "Alpha": alpha, + "EmissionExps": emission_exps, + "TransitionExps": transition_exps, + "LogLikelihood": log_likelihood + } + + def setUp(self): + self.op_type = "linear_chain_crf" + self.set_test_data() + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["Emission", "Transition"], "LogLikelihood") + + def test_check_grad_ignore_transition(self): + self.check_grad( + ["Emission"], "LogLikelihood", no_grad_set=set("Transition")) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_rank_table.py b/python/paddle/v2/framework/tests/test_lod_rank_table.py new file mode 100644 index 0000000000000000000000000000000000000000..2242d4391dc7c12e59f4f157b34b0525efcbc9a1 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_rank_table.py @@ -0,0 +1,29 @@ +from paddle.v2.framework.layers import lod_rank_table, data +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.framework import g_main_program +import paddle.v2.framework.core as core +import numpy +import unittest + + +class TestLoDRankTable(unittest.TestCase): + def test_lod_rank_table(self): + x = data(name='x', shape=[100]) + cpu = core.CPUPlace() + rank_table = lod_rank_table(x=x, level=1) + rank_table.persistable = True + exe = Executor(cpu) + scope = core.Scope() + + tensor = core.LoDTensor() + tensor.set(numpy.random.random(size=(17, 100)), cpu) + tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]]) + + exe.run(g_main_program, scope=scope, feed={'x': tensor}) + var = scope.find_var(rank_table.name) + table = var.get_lod_rank_table() + self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_tensor_array.py b/python/paddle/v2/framework/tests/test_lod_tensor_array.py new file mode 100644 index 0000000000000000000000000000000000000000..a433bcf622b14a1d2d33b5b98d555e1a21e4b9e8 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_tensor_array.py @@ -0,0 +1,38 @@ +import unittest +import paddle.v2.framework.core as core +import numpy + + +class TestLoDTensorArray(unittest.TestCase): + def test_get_set(self): + scope = core.Scope() + arr = scope.var('tmp_lod_tensor_array') + tensor_array = arr.get_lod_tensor_array() + self.assertEqual(0, len(tensor_array)) + cpu = core.CPUPlace() + for i in xrange(10): + t = core.LoDTensor() + t.set(numpy.array([i], dtype='float32'), cpu) + t.set_lod([[0, 1]]) + tensor_array.append(t) + + self.assertEqual(10, len(tensor_array)) + + for i in xrange(10): + t = tensor_array[i] + self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32')) + self.assertEqual([[0, 1]], t.lod()) + + t = core.LoDTensor() + t.set(numpy.array([i + 10], dtype='float32'), cpu) + t.set_lod([[0, 2]]) + tensor_array[i] = t + t = tensor_array[i] + self.assertEqual( + numpy.array(t), numpy.array( + [i + 10], dtype='float32')) + self.assertEqual([[0, 2]], t.lod()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lstm_op.py b/python/paddle/v2/framework/tests/test_lstm_op.py index 93a4e450e916716e27573d192bace73f271733de..ff75160083f2936dd653a8396254bf16d1752ffa 100644 --- a/python/paddle/v2/framework/tests/test_lstm_op.py +++ b/python/paddle/v2/framework/tests/test_lstm_op.py @@ -52,7 +52,7 @@ def lstm( g = np.dot(h_pre, w_h) # 1 x 4D g = g + x g = np.reshape(g, (1, g.size)) - c_tmp, g_i, g_f, g_o = np.split(g, 4, axis=1) + c, g_i, g_f, g_o = np.split(g, 4, axis=1) if w_c is None: g_i = act_gate(g_i) # 1 x D g_f = act_gate(g_f) # 1 x D @@ -60,7 +60,7 @@ def lstm( w_ic, w_fc, w_oc = np.split(w_c, 3, axis=1) g_i = act_gate(g_i + w_ic * c_pre) # 1 x D g_f = act_gate(g_f + w_fc * c_pre) # 1 x D - c = g_f * c_pre + g_i * act_cand(c_tmp) # 1 x D + c = g_f * c_pre + g_i * act_cand(c) # 1 x D if w_c is None: g_o = act_gate(g_o) # 1 x D @@ -68,8 +68,7 @@ def lstm( _, _, w_oc = np.split(w_c, 3, axis=1) g_o = act_gate(g_o + w_oc * c) # 1 x D h = g_o * act_cell(c) - bg = np.concatenate((act_cand(c_tmp), g_i, g_f, g_o), axis=1) - return h, c, bg + return h, c def _reverse(x, lod): y = np.zeros_like(x) @@ -82,7 +81,6 @@ def lstm( batch_size = len(offset) - 1 hidden = [] cell = [] - gate = [] input = _reverse(input, offset) if is_reverse else input if w_b is not None: input = input + np.tile(w_b, (offset[-1], 1)) @@ -94,96 +92,109 @@ def lstm( c_pre = c0[i] # 1 x D for j in range(seq_len): # compute one step - h_pre, c_pre, g_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate, - act_cell, act_cand) + h_pre, c_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate, + act_cell, act_cand) hidden.append(h_pre.flatten()) cell.append(c_pre.flatten()) - gate.append(g_pre.flatten()) - hidden = np.array(hidden).astype("float64") - cell = np.array(cell).astype("float64") - gate = np.array(gate).astype("float64") + hidden = np.array(hidden).astype('float64') + cell = np.array(cell).astype('float64') hidden = _reverse(hidden, offset) if is_reverse else hidden cell = _reverse(cell, offset) if is_reverse else cell - assert gate.shape == input.shape assert hidden.shape == (input.shape[0], input.shape[1] / 4) assert cell.shape == (input.shape[0], input.shape[1] / 4) - return hidden, cell, gate + return hidden, cell class TestLstmOp(OpTest): - def set_data(self): - self.lod = [[0, 2, 6, 9]] - self.D = 64 - self.sort_idx = [2, 6, 0, 3, 7, 1, 4, 8, 5] + def set_argument(self): + self.lod = [[0, 2, 5, 7]] + self.D = 16 - self.act_gate = "sigmoid" - self.act_cell = "tanh" - self.act_cand = "tanh" + self.act_gate = 'sigmoid' + self.act_cell = 'tanh' + self.act_cand = 'tanh' + self.has_initial_state = True self.is_reverse = False def setUp(self): - self.set_data() - self.op_type = "lstm" + self.set_argument() + self.op_type = 'lstm' T = self.lod[0][-1] N = len(self.lod[0]) - 1 - x = np.random.normal(size=(T, 4 * self.D)).astype("float64") - h0 = np.zeros((N, self.D)).astype("float64") - c0 = np.zeros((N, self.D)).astype("float64") - w = np.random.normal(size=(self.D, 4 * self.D)).astype("float64") - b = np.random.normal(size=(1, 7 * self.D)).astype("float64") + x = np.random.normal(size=(T, 4 * self.D)).astype('float64') + h0 = np.zeros((N, self.D)).astype('float64') + c0 = np.zeros((N, self.D)).astype('float64') + w = np.random.normal(size=(self.D, 4 * self.D)).astype('float64') + b = np.random.normal(size=(1, 7 * self.D)).astype('float64') w_b = b[:, 0:4 * self.D] w_c = b[:, 4 * self.D:] - h, c, g = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, - ACTVATION[self.act_gate], ACTVATION[self.act_cell], - ACTVATION[self.act_cand]) - - g_sort = np.zeros_like(x) - for i, j in enumerate(self.sort_idx): - g_sort[i, :] = g[j, :] - - self.inputs = { - 'Input': (x, self.lod), - 'H0': h0, - 'C0': c0, - 'Weight': w, - 'Bias': b - } + h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, + ACTVATION[self.act_gate], ACTVATION[self.act_cell], + ACTVATION[self.act_cand]) + + self.inputs = {'Input': (x, self.lod), 'Weight': w, 'Bias': b} + if self.has_initial_state: + self.inputs['H0'] = h0 + self.inputs['C0'] = c0 + self.outputs = { 'Hidden': (h, self.lod), 'Cell': (c, self.lod), - 'BatchGate': g_sort } self.attrs = { 'usePeepholes': True, 'isReverse': self.is_reverse, - 'gateActivation': 'sigmoid', - 'cellActivation': 'tanh', - 'candidateActivation': 'tanh' + 'gateActivation': self.act_gate, + 'cellActivation': self.act_cell, + 'candidateActivation': self.act_cand } def test_check_output(self): - self.check_output() + self.check_output(atol=1e-8) + + #TODO(qingqing) add more unit testing case + def test_check_grad(self): + # TODO(qingqing) remove folowing lines after the check_grad is refined. + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias'], ['Hidden'], max_relative_error=5e-4) + + +class TestLstmOpHasNoInitial(TestLstmOp): + def set_argument(self): + self.lod = [[0, 2, 5, 7]] + self.D = 16 + + self.act_gate = 'sigmoid' + self.act_cell = 'tanh' + self.act_cand = 'tanh' + + self.has_initial_state = False + self.is_reverse = True class TestLstmOpRerverse(TestLstmOp): - def set_data(self): - self.lod = [[0, 2, 6, 9]] - self.D = 64 - self.sort_idx = [2, 6, 0, 3, 7, 1, 4, 8, 5] + def set_argument(self): + self.lod = [[0, 2, 5, 7]] + self.D = 16 - self.act_gate = "sigmoid" - self.act_cell = "tanh" - self.act_cand = "tanh" + self.act_gate = 'sigmoid' + self.act_cell = 'tanh' + self.act_cand = 'tanh' + self.has_initial_state = True self.is_reverse = True -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_operator_desc.py b/python/paddle/v2/framework/tests/test_operator_desc.py index 7355f72455ca4f821c9520d97162e3e0050383af..a0bc4e0b91602cfc90f91a1e2dd4bce22c0dbf6d 100644 --- a/python/paddle/v2/framework/tests/test_operator_desc.py +++ b/python/paddle/v2/framework/tests/test_operator_desc.py @@ -1,11 +1,11 @@ import unittest -from paddle.v2.framework.framework import Variable, Program, g_program +from paddle.v2.framework.framework import Variable, Program, g_main_program import paddle.v2.framework.core as core class TestOperator(unittest.TestCase): def test_error_type(self): - block = g_program.create_block() + block = g_main_program.create_block() try: block.append_op() self.assertFail() diff --git a/python/paddle/v2/framework/tests/test_optimizer.py b/python/paddle/v2/framework/tests/test_optimizer.py index 45396c9bec9ccf0668b048b2b4855d7a665ebea5..9333df8f7f347a080cfb035ccd0c575ded7c423a 100644 --- a/python/paddle/v2/framework/tests/test_optimizer.py +++ b/python/paddle/v2/framework/tests/test_optimizer.py @@ -7,6 +7,7 @@ from paddle.v2.framework.backward import append_backward_ops class TestOptimizer(unittest.TestCase): def test_sgd_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -22,12 +23,13 @@ class TestOptimizer(unittest.TestCase): outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) - opts = sgd_optimizer.minimize(mul_out) + opts = sgd_optimizer.minimize(mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") def test_sgd_optimizer_with_global_step(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -44,15 +46,22 @@ class TestOptimizer(unittest.TestCase): attrs={"x_num_col_dims": 1}) global_step = block.create_var( dtype="float32", shape=[1], lod_level=0, name="step") + learning_rate = 0.01 sgd_optimizer = optimizer.SGDOptimizer( - learning_rate=0.01, global_step=global_step) - opts = sgd_optimizer.minimize(mul_out) + learning_rate=learning_rate, global_step=global_step) + opts = sgd_optimizer.minimize(mul_out, init_program) self.assertEqual(len(opts), 2) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") increment_op = opts[1] self.assertEqual(increment_op.type, "increment") + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 1) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + class TestMomentumOptimizer(unittest.TestCase): class MockMomentum(optimizer.MomentumOptimizer): @@ -63,6 +72,7 @@ class TestMomentumOptimizer(unittest.TestCase): return self._velocity_acc_str def test_vanilla_momentum_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -77,12 +87,14 @@ class TestMomentumOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) - momentum_optimizer = self.MockMomentum(learning_rate=0.01, momentum=0.2) + learning_rate = 0.01 + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) - opts = momentum_optimizer.create_optimization_pass(params_grads, - mul_out) + opts = momentum_optimizer.create_optimization_pass( + params_grads, mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") @@ -96,7 +108,16 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + def test_nesterov_momentum_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -111,13 +132,14 @@ class TestMomentumOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 momentum_optimizer = self.MockMomentum( - learning_rate=0.01, momentum=0.2, use_nesterov=True) + learning_rate=learning_rate, momentum=0.2, use_nesterov=True) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) - opts = momentum_optimizer.create_optimization_pass(params_grads, - mul_out) + opts = momentum_optimizer.create_optimization_pass( + params_grads, mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") @@ -131,6 +153,14 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + class TestAdagradOptimizer(unittest.TestCase): class MockAdagrad(optimizer.AdagradOptimizer): @@ -141,6 +171,7 @@ class TestAdagradOptimizer(unittest.TestCase): return self._moment_acc_str def test_adagrad_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -155,11 +186,14 @@ class TestAdagradOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) - adagrad_optimizer = self.MockAdagrad(learning_rate=0.01, epsilon=1.0e-6) + learning_rate = 0.01 + adagrad_optimizer = self.MockAdagrad( + learning_rate=learning_rate, epsilon=1.0e-6) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0) - opts = adagrad_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adagrad_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 1) adagrad_op = opts[0] self.assertEqual(adagrad_op.type, "adagrad") @@ -172,6 +206,14 @@ class TestAdagradOptimizer(unittest.TestCase): self.assertEqual(len(moment_acc), 1) self.assertTrue(mul_x.name in moment_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + class TestAdamOptimizer(unittest.TestCase): class MockAdam(optimizer.AdamOptimizer): @@ -185,6 +227,7 @@ class TestAdamOptimizer(unittest.TestCase): return self._moment2_acc_str def test_adam_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -199,12 +242,14 @@ class TestAdamOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 adam_optimizer = self.MockAdam( - learning_rate=0.01, beta1=0.9, beta2=0.999) + learning_rate=learning_rate, beta1=0.9, beta2=0.999) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adam_optimizer.get_accumulators()), 0) - opts = adam_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adam_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 3) adam_op = opts[0] self.assertEqual(adam_op.type, "adam") @@ -221,6 +266,12 @@ class TestAdamOptimizer(unittest.TestCase): self.assertTrue(mul_x.name in moment1_acc) self.assertTrue(mul_x.name in moment2_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 5) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + class TestAdamaxOptimizer(unittest.TestCase): class MockAdamax(optimizer.AdamaxOptimizer): @@ -234,6 +285,7 @@ class TestAdamaxOptimizer(unittest.TestCase): return self._inf_norm_acc_str def test_adamax_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -248,12 +300,14 @@ class TestAdamaxOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 adamax_optimizer = self.MockAdamax( - learning_rate=0.01, beta1=0.9, beta2=0.999) + learning_rate=learning_rate, beta1=0.9, beta2=0.999) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adamax_optimizer.get_accumulators()), 0) - opts = adamax_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adamax_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 2) adam_op = opts[0] self.assertEqual(adam_op.type, "adamax") @@ -270,6 +324,12 @@ class TestAdamaxOptimizer(unittest.TestCase): self.assertTrue(mul_x.name in moment_acc) self.assertTrue(mul_x.name in inf_norm_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 4) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_parameter.py b/python/paddle/v2/framework/tests/test_parameter.py index 1ac0cdd99f1b7c15d64ae9d2c465d5a9d563bd80..f04eb4cf27276b0f7da0793c97742ac42e4583be 100644 --- a/python/paddle/v2/framework/tests/test_parameter.py +++ b/python/paddle/v2/framework/tests/test_parameter.py @@ -1,11 +1,11 @@ import unittest -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program import paddle.v2.framework.core as core class TestParameter(unittest.TestCase): def test_param(self): - b = g_program.create_block() + b = g_main_program.create_block() param = b.create_parameter( name='fc.w', shape=[784, 100], diff --git a/python/paddle/v2/framework/tests/test_precision_recall_op.py b/python/paddle/v2/framework/tests/test_precision_recall_op.py new file mode 100644 index 0000000000000000000000000000000000000000..d3dbdb6e2aba6dfe98440ad07083cf1ffda5b668 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_precision_recall_op.py @@ -0,0 +1,173 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def calc_precision(tp_count, fp_count): + if tp_count > 0.0 or fp_count > 0.0: + return tp_count / (tp_count + fp_count) + return 1.0 + + +def calc_recall(tp_count, fn_count): + if tp_count > 0.0 or fn_count > 0.0: + return tp_count / (tp_count + fn_count) + return 1.0 + + +def calc_f1_score(precision, recall): + if precision > 0.0 or recall > 0.0: + return 2 * precision * recall / (precision + recall) + return 0.0 + + +def get_states(idxs, labels, cls_num, weights=None): + ins_num = idxs.shape[0] + # TP FP TN FN + states = np.zeros((cls_num, 4)).astype('float32') + for i in xrange(ins_num): + w = weights[i] if weights is not None else 1.0 + idx = idxs[i][0] + label = labels[i][0] + if idx == label: + states[idx][0] += w + for j in xrange(cls_num): + states[j][2] += w + states[idx][2] -= w + else: + states[label][3] += w + states[idx][1] += w + for j in xrange(cls_num): + states[j][2] += w + states[label][2] -= w + states[idx][2] -= w + return states + + +def compute_metrics(states, cls_num): + total_tp_count = 0.0 + total_fp_count = 0.0 + total_fn_count = 0.0 + macro_avg_precision = 0.0 + macro_avg_recall = 0.0 + for i in xrange(cls_num): + total_tp_count += states[i][0] + total_fp_count += states[i][1] + total_fn_count += states[i][3] + macro_avg_precision += calc_precision(states[i][0], states[i][1]) + macro_avg_recall += calc_recall(states[i][0], states[i][3]) + metrics = [] + macro_avg_precision /= cls_num + macro_avg_recall /= cls_num + metrics.append(macro_avg_precision) + metrics.append(macro_avg_recall) + metrics.append(calc_f1_score(macro_avg_precision, macro_avg_recall)) + micro_avg_precision = calc_precision(total_tp_count, total_fp_count) + metrics.append(micro_avg_precision) + micro_avg_recall = calc_recall(total_tp_count, total_fn_count) + metrics.append(micro_avg_recall) + metrics.append(calc_f1_score(micro_avg_precision, micro_avg_recall)) + return np.array(metrics).astype('float32') + + +class TestPrecisionRecallOp_0(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + states = get_states(idxs, labels, cls_num) + metrics = compute_metrics(states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = {'MaxProbs': max_probs, 'Indices': idxs, 'Labels': labels} + + self.outputs = { + 'BatchMetrics': metrics, + 'AccumMetrics': metrics, + 'AccumStatesInfo': states + } + + def test_check_output(self): + self.check_output() + + +class TestPrecisionRecallOp_1(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + + states = get_states(idxs, labels, cls_num, weights) + metrics = compute_metrics(states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = { + 'MaxProbs': max_probs, + 'Indices': idxs, + 'Labels': labels, + 'Weights': weights + } + + self.outputs = { + 'BatchMetrics': metrics, + 'AccumMetrics': metrics, + 'AccumStatesInfo': states + } + + def test_check_output(self): + self.check_output() + + +class TestPrecisionRecallOp_2(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + states = np.random.randint(0, 30, (cls_num, 4)).astype('float32') + + accum_states = get_states(idxs, labels, cls_num, weights) + batch_metrics = compute_metrics(accum_states, cls_num) + accum_states += states + accum_metrics = compute_metrics(accum_states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = { + 'MaxProbs': max_probs, + 'Indices': idxs, + 'Labels': labels, + 'Weights': weights, + 'StatesInfo': states + } + + self.outputs = { + 'BatchMetrics': batch_metrics, + 'AccumMetrics': accum_metrics, + 'AccumStatesInfo': accum_states + } + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_program.py b/python/paddle/v2/framework/tests/test_program.py index be020573b7dcd9f8dcd0f99d654dc8b2106abb2b..7be67b6614ee3302a319289b821a214a81b6f64e 100644 --- a/python/paddle/v2/framework/tests/test_program.py +++ b/python/paddle/v2/framework/tests/test_program.py @@ -2,35 +2,35 @@ import unittest import paddle.v2.framework.core as core from paddle.v2.framework.framework import Program -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program class TestProgram(unittest.TestCase): def test_program(self): - b = g_program.current_block() + b = g_main_program.current_block() self.assertEqual(-1, b.parent_idx) self.assertEqual(0, b.idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(2, b.idx) self.assertEqual(1, b.parent_idx) - g_program.rollback() + g_main_program.rollback() - b = g_program.current_block() + b = g_main_program.current_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(3, b.idx) self.assertEqual(1, b.parent_idx) - g_program.rollback() - b = g_program.current_block() + g_main_program.rollback() + b = g_main_program.current_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py b/python/paddle/v2/framework/tests/test_recognize_digits_conv.py index a9b6c8410e2af36e6928b2fac919398473611728..c3186e25b37fe112987017521daa336199085134 100644 --- a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py +++ b/python/paddle/v2/framework/tests/test_recognize_digits_conv.py @@ -4,26 +4,26 @@ import paddle.v2.framework.nets as nets import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program, g_main_program from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() images = layers.data( name='pixel', shape=[1, 28, 28], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) label = layers.data( name='label', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, @@ -31,8 +31,8 @@ conv_pool_1 = nets.simple_img_conv_pool( pool_size=2, pool_stride=2, act="relu", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) conv_pool_2 = nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -40,23 +40,33 @@ conv_pool_2 = nets.simple_img_conv_pool( pool_size=2, pool_stride=2, act="relu", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) predict = layers.fc(input=conv_pool_2, size=10, act="softmax", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program) + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean(x=cost, main_program=main_program) +accuracy = layers.accuracy( + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +# optimizer = optimizer.MomentumOptimizer(learning_rate=0.1 / 128.0, +# momentum=0.9) +optimizer = optimizer.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.999) +opts = optimizer.minimize(avg_cost, startup_program) BATCH_SIZE = 50 -PASS_NUM = 1 +PASS_NUM = 3 train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), @@ -65,7 +75,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) for pass_id in range(PASS_NUM): count = 0 @@ -80,13 +90,14 @@ for pass_id in range(PASS_NUM): tensor_img.set(img_data, place) tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={"pixel": tensor_img, "label": tensor_y}, - fetch_list=[avg_cost]) - + fetch_list=[avg_cost, accuracy]) loss = np.array(outs[0]) + acc = np.array(outs[1]) - if loss < 10.0: - exit(0) # if avg cost less than 10.0, we think our code is good. + if loss < 10.0 and acc > 0.9: + # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good. + exit(0) exit(1) diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py b/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py index a8a34b2a952c8d374089ab8142b530610b2afe59..076cf882160cd53f45ef291d82ba57ada843a287 100644 --- a/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py +++ b/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py @@ -3,65 +3,72 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor from paddle.v2.framework.regularizer import L2DecayRegularizer +from paddle.v2.framework.initializer import UniformInitializer import numpy as np BATCH_SIZE = 128 -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() image = layers.data( name='x', shape=[784], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) param_attr = { 'name': None, - 'init_attr': { - 'type': 'uniform_random', - 'min': -1.0, - 'max': 1.0 - }, + 'initializer': UniformInitializer( + low=-1.0, high=1.0), 'regularization': L2DecayRegularizer(0.0005 * BATCH_SIZE) } hidden1 = layers.fc(input=image, size=128, act='relu', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) hidden2 = layers.fc(input=hidden1, size=64, act='relu', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) predict = layers.fc(input=hidden2, size=10, act='softmax', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) label = layers.data( name='y', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) - -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) +accuracy = layers.accuracy( + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) + +optimizer = optimizer.MomentumOptimizer(learning_rate=0.001, momentum=0.9) +opts = optimizer.minimize(avg_cost, startup_program) train_reader = paddle.batch( paddle.reader.shuffle( @@ -71,7 +78,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): @@ -86,11 +93,12 @@ for pass_id in range(PASS_NUM): tensor_y = core.LoDTensor() tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={'x': tensor_x, 'y': tensor_y}, - fetch_list=[avg_cost]) + fetch_list=[avg_cost, accuracy]) out = np.array(outs[0]) + acc = np.array(outs[1]) if out[0] < 5.0: exit(0) # if avg cost less than 5.0, we think our code is good. exit(1) diff --git a/python/paddle/v2/framework/tests/test_recommender_system.py b/python/paddle/v2/framework/tests/test_recommender_system.py new file mode 100644 index 0000000000000000000000000000000000000000..7e54f0d1b86461954b05d1e3c1d5f49f71ff9068 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_recommender_system.py @@ -0,0 +1,315 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program, g_main_program +from paddle.v2.framework.executor import Executor + +import numpy as np + +startup_program = Program() +main_program = Program() +is_sparse = True +use_gpu = False +BATCH_SIZE = 256 + + +def get_usr_combined_features(): + # FIXME(dzh) : old API integer_value(10) may has range check. + # currently we don't have user configurated check. + + USR_DICT_SIZE = paddle.dataset.movielens.max_user_id() + 1 + + uid = layers.data( + name='user_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + usr_emb = layers.embedding( + input=uid, + data_type='float32', + size=[USR_DICT_SIZE, 32], + param_attr={'name': 'user_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_fc = layers.fc(input=usr_emb, + size=32, + main_program=main_program, + startup_program=startup_program) + + USR_GENDER_DICT_SIZE = 2 + + usr_gender_id = layers.data( + name='gender_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + usr_gender_emb = layers.embedding( + input=usr_gender_id, + size=[USR_GENDER_DICT_SIZE, 16], + param_attr={'name': 'gender_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_gender_fc = layers.fc(input=usr_gender_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + USR_AGE_DICT_SIZE = len(paddle.dataset.movielens.age_table) + usr_age_id = layers.data( + name='age_id', + shape=[1], + data_type="int64", + main_program=main_program, + startup_program=startup_program) + + usr_age_emb = layers.embedding( + input=usr_age_id, + size=[USR_AGE_DICT_SIZE, 16], + is_sparse=is_sparse, + param_attr={'name': 'age_table'}, + main_program=main_program, + startup_program=startup_program) + + usr_age_fc = layers.fc(input=usr_age_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + USR_JOB_DICT_SIZE = paddle.dataset.movielens.max_job_id() + 1 + usr_job_id = layers.data( + name='job_id', + shape=[1], + data_type="int64", + main_program=main_program, + startup_program=startup_program) + + usr_job_emb = layers.embedding( + input=usr_job_id, + size=[USR_JOB_DICT_SIZE, 16], + param_attr={'name': 'job_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_job_fc = layers.fc(input=usr_job_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + concat_embed = layers.concat( + input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], + axis=1, + main_program=main_program, + startup_program=startup_program) + + usr_combined_features = layers.fc(input=concat_embed, + size=200, + act="tanh", + main_program=main_program, + startup_program=startup_program) + + return usr_combined_features + + +def get_mov_combined_features(): + + MOV_DICT_SIZE = paddle.dataset.movielens.max_movie_id() + 1 + + mov_id = layers.data( + name='movie_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_emb = layers.embedding( + input=mov_id, + data_type='float32', + size=[MOV_DICT_SIZE, 32], + param_attr={'name': 'movie_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_fc = layers.fc(input=mov_emb, + size=32, + main_program=main_program, + startup_program=startup_program) + + CATEGORY_DICT_SIZE = len(paddle.dataset.movielens.movie_categories()) + + category_id = layers.data( + name='category_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_categories_emb = layers.embedding( + input=category_id, + size=[CATEGORY_DICT_SIZE, 32], + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_categories_hidden = layers.sequence_pool( + input=mov_categories_emb, + pool_type="sum", + main_program=main_program, + startup_program=startup_program) + + MOV_TITLE_DICT_SIZE = len(paddle.dataset.movielens.get_movie_title_dict()) + + mov_title_id = layers.data( + name='movie_title', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_title_emb = layers.embedding( + input=mov_title_id, + size=[MOV_TITLE_DICT_SIZE, 32], + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_title_conv = nets.sequence_conv_pool( + input=mov_title_emb, + num_filters=32, + filter_size=3, + act="tanh", + pool_type="sum", + main_program=main_program, + startup_program=startup_program) + + concat_embed = layers.concat( + input=[mov_fc, mov_categories_hidden, mov_title_conv], + axis=1, + main_program=main_program, + startup_program=startup_program) + + # FIXME(dzh) : need tanh operator + mov_combined_features = layers.fc(input=concat_embed, + size=200, + act="tanh", + main_program=main_program, + startup_program=startup_program) + + return mov_combined_features + + +def model(): + usr_combined_features = get_usr_combined_features() + mov_combined_features = get_mov_combined_features() + + # need cos sim + inference = layers.cos_sim( + X=usr_combined_features, + Y=mov_combined_features, + main_program=main_program, + startup_program=startup_program) + + label = layers.data( + name='score', + shape=[1], + data_type='float32', + main_program=main_program, + startup_program=startup_program) + + square_cost = layers.square_error_cost( + input=inference, + label=label, + main_program=main_program, + startup_program=startup_program) + + avg_cost = layers.mean( + x=square_cost, + main_program=main_program, + startup_program=startup_program) + + return avg_cost + + +def main(): + cost = model() + sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.2) + opts = sgd_optimizer.minimize(cost, startup_program=startup_program) + block = main_program.block(0) + + if use_gpu: + place = core.GPUPlace(0) + else: + place = core.CPUPlace() + + exe = Executor(place) + exe.run(startup_program, feed={}, fetch_list=[]) + + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.movielens.train(), buf_size=8192), + batch_size=BATCH_SIZE) + + feeding = { + 'user_id': 0, + 'gender_id': 1, + 'age_id': 2, + 'job_id': 3, + 'movie_id': 4, + 'category_id': 5, + 'movie_title': 6, + 'score': 7 + } + + def func_feed(feeding, data): + feed_tensors = {} + for (key, idx) in feeding.iteritems(): + tensor = core.LoDTensor() + if key != "category_id" and key != "movie_title": + if key == "score": + numpy_data = np.array(map(lambda x: x[idx], data)).astype( + "float32") + else: + numpy_data = np.array(map(lambda x: x[idx], data)).astype( + "int64") + else: + numpy_data = map(lambda x: np.array(x[idx]).astype("int64"), + data) + lod_info = [len(item) for item in numpy_data] + offset = 0 + lod = [offset] + for item in lod_info: + offset += item + lod.append(offset) + numpy_data = np.concatenate(numpy_data, axis=0) + tensor.set_lod([lod]) + + numpy_data = numpy_data.reshape([numpy_data.shape[0], 1]) + tensor.set(numpy_data, place) + feed_tensors[key] = tensor + return feed_tensors + + PASS_NUM = 100 + for pass_id in range(PASS_NUM): + for data in train_reader(): + outs = exe.run(main_program, + feed=func_feed(feeding, data), + fetch_list=[cost]) + out = np.array(outs[0]) + if out[0] < 6.0: + # if avg cost less than 6.0, we think our code is good. + exit(0) + + +main() diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 6c9081a7c37d2a68c50b5748c87199efe9a90cc7..001de349d1f62e661335e63f7519c59697a1030c 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -1,51 +1,67 @@ -import logging -import paddle.v2.framework.core as core import unittest -import numpy as np -from paddle.v2.framework.op import Operator, RecurrentOp -from op_test import get_numeric_gradient - -def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) +import logging +from op_test import get_numeric_gradient +from paddle.v2.framework.layers import * +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +import numpy as np +import paddle.v2.framework.core as core -class PySimpleRNN(object): - ''' - A simple implementation of RNN based on numpy, to futhur test RecurrentOp's alogorithm - ''' - def __init__(self, input_dim=30, batch_size=50, weight_dim=15, sent_len=11): - self.x = np.random.normal(size=(sent_len, batch_size, - input_dim)).astype("float32") - self.W = np.random.normal(size=(input_dim, input_dim)).astype("float32") - self.U = np.random.normal(size=(input_dim, input_dim)).astype("float32") - self.h_boot = np.random.normal(size=(batch_size, - input_dim)).astype("float32") +class PyRNNBase(object): + def __init__(self, input_shape, output_shape): + self.x = np.ones(shape=input_shape).astype("float32") + self.y = np.zeros(shape=output_shape).astype("float32") - # memories - self.mems = [ - np.zeros(shape=(batch_size, input_dim)).astype("float32") - for i in range(sent_len) - ] + def step(self): + pass def forward(self): - xs = self.segment_inputs() for step_id in range(self.x.shape[0]): - self.step(step_id, xs[step_id]) - return self.concat_outputs() + self.step(step_id, self.x[step_id]) + return np.array([np.mean(self.y)]) def segment_inputs(self): return [self.x[i] for i in range(self.x.shape[0])] - def concat_outputs(self): - return np.array(self.mems).astype("float32") + +class PySimpleRNN1(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(PySimpleRNN1, self).__init__(input_shape, output_shape) + + seq_len, batch_size, input_dim = input_shape + self.h_boot = np.random.normal(size=(batch_size, + input_dim)).astype("float32") + + self.scale = 1.0 / 2.0 + men_dim = (seq_len, batch_size, input_dim) + self.mems = np.zeros(shape=men_dim).astype("float32") + + def step(self, step_id, x): + if step_id == 0: + pre_mem = self.h_boot + else: + pre_mem = self.mems[step_id - 1] + self.mems[step_id] = (pre_mem + x) * self.scale + self.y[step_id] = self.mems[step_id] + + +class PySimpleRNN2(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(PySimpleRNN2, self).__init__(input_shape, output_shape) + + seq_len, batch_size, input_dim = input_shape + self.W = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.U = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.h_boot = np.ones(shape=(batch_size, input_dim)).astype("float32") + + men_dim = (seq_len, batch_size, input_dim) + self.mems = np.zeros(shape=men_dim).astype("float32") def step(self, step_id, x): - ''' - run a step - ''' - mem = self.mems[step_id] if step_id > 0: pre_mem = self.mems[step_id - 1] else: @@ -53,108 +69,128 @@ class PySimpleRNN(object): xW = np.matmul(x, self.W).astype("float32") hU = np.matmul(pre_mem, self.U).astype("float32") - sum = xW + hU - self.mems[step_id] = py_sigmoid(sum) - + def py_sigmoid(x): + return 1. / (1. + np.exp(-x)) -class PySimpleRNNTest(unittest.TestCase): - def setUp(self): - self.rnn = PySimpleRNN() - - def test_forward(self): - output = self.rnn.forward() + self.mems[step_id] = py_sigmoid(xW + hU) + self.y[step_id] = self.mems[step_id] -def create_tensor(scope, name, shape, np_data): - tensor = scope.var(name).get_tensor() - tensor.set_dims(shape) - tensor.set(np_data, core.CPUPlace()) +def create_tensor(np_data, place): + tensor = core.LoDTensor() + tensor.set(np_data, place) return tensor -class RecurrentOpTest(unittest.TestCase): +class RecurrentOpTest1(unittest.TestCase): ''' Test RNNOp - equation: - h_t = \sigma (W x_t + U h_{t-1}) - weights: - - W - - U + h_t = ( x_t + h_{t-1} ) / scale vars: - x memories: - h outputs: - - h + - h ''' - input_dim = 30 - batch_size = 50 - weight_dim = 15 - sent_len = 11 + input_dim = 2 + batch_size = 1 + sent_len = 1 + + def setup_program(self): + self.main_program = Program() + self.startup_program = Program() + self.p_info = { + "main_program": self.main_program, + "startup_program": self.startup_program + } + self.place = core.CPUPlace() def setUp(self): - self.py_rnn = PySimpleRNN(self.input_dim, self.batch_size, - self.weight_dim, self.sent_len) + self.setup_program() + self.data_field = {"x", "h_boot"} - def forward(self): - self.scope = core.Scope() - self.create_global_variables() - self.create_rnn_op() - self.create_step_net() - ctx = core.DeviceContext.create(core.CPUPlace()) - self.rnnop.run(self.scope, ctx) - return np.array(self.scope.find_var("h@mem").get_tensor()).astype( - "float32") - - def create_global_variables(self): - # create inlink - x_np_data = self.py_rnn.x - create_tensor(self.scope, "x", - [self.sent_len, self.batch_size, self.input_dim], - x_np_data) - W_np_data = self.py_rnn.W - create_tensor(self.scope, "W", [self.input_dim, self.input_dim], - W_np_data) - - U_np_data = self.py_rnn.U - create_tensor(self.scope, "U", [self.input_dim, self.input_dim], - U_np_data) - - h_boot_np_data = self.py_rnn.h_boot - create_tensor(self.scope, "h_boot", [self.batch_size, self.input_dim], - h_boot_np_data) - self.scope.var("step_scopes") - self.scope.var("h@mem") + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) + + self.output = mean(x=self.create_rnn_op(), **self.p_info) def create_rnn_op(self): - # create RNNOp - self.rnnop = RecurrentOp( - # inputs - inputs=["x"], - initial_states=["h_boot"], - step_net="stepnet", - # outputs - outputs=["h@mem"], - step_scopes="step_scopes", - # attributes - ex_states=["h@pre"], - states=["h@mem"]) - - def create_step_net(self): - stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x", Y="W", Out="Wx") - h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@mem") - - for op in [x_fc_op, h_fc_op, sum_op, sig_op]: - stepnet.append_op(op) - stepnet.complete_add_op(True) - self.rnnop.set_stepnet(stepnet) - - def test_forward(self): + x = data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot = data( + shape=[self.input_dim], + data_type='float32', + name='h_boot', + **self.p_info) + h_boot.stop_gradient = False + + rnn = StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre = rnn.memory(init=h_boot) + x_t = rnn.step_input(x) + + h = scale( + x=elementwise_add( + x=h_pre, y=x_t, **self.p_info), + scale=self.py_rnn.scale, + **self.p_info) + + rnn.update_memory(h_pre, h) + rnn.output(h) + + return rnn() + + def forward(self): + self.feed_map = { + x: create_tensor(getattr(self.py_rnn, x), self.place) + for x in self.data_field + } + exe = Executor(self.place) + out = exe.run(self.main_program, + feed=self.feed_map, + fetch_list=[self.output]) + + return np.array(out[0]) + + def backward(self): + self.feed_map = { + x: create_tensor(getattr(self.py_rnn, x), self.place) + for x in self.data_field + } + fetch_list = [ + self.main_program.global_block().var(x + "@GRAD") + for x in self.data_field + ] + + exe = Executor(self.place) + return exe.run(self.main_program, + feed=self.feed_map, + fetch_list=fetch_list) + + def test_backward(self): + self.check_forward() + + append_backward_ops(self.output) + + ana_grad = [np.array(x) for x in self.backward()] + + num_grad = self.get_numerical_gradient() + for idx, name in enumerate(self.data_field): + self.assertEqual(num_grad[idx].shape, ana_grad[idx].shape) + self.assertTrue( + np.isclose( + num_grad[idx], ana_grad[idx], rtol=0.1).all()) + + def check_forward(self): print 'test recurrent op forward' pd_output = self.forward() py_output = self.py_rnn.forward() @@ -164,44 +200,195 @@ class RecurrentOpTest(unittest.TestCase): self.assertEqual(pd_output.shape, py_output.shape) self.assertTrue(np.isclose(pd_output, py_output, rtol=0.1).all()) + def get_numerical_gradient(self, delta=0.005): + dloss_dout = 1.0 + feed_list = [getattr(self.py_rnn, x) for x in self.data_field] + grad_list = [np.zeros_like(x) for x in feed_list] + for feed, grad in zip(feed_list, grad_list): + for f, g in np.nditer([feed, grad], op_flags=['readwrite']): + o = float(f) + f[...] = o + delta + y_pos = self.forward() -class RecurrentGradientOpTest(unittest.TestCase): - def create_forward_op(self): - self.forward_op = RecurrentOp( - # inputs - inputs=["x"], - initial_states=["h_boot"], - step_net="stepnet", - # outputs - outputs=["h"], - step_scopes="step_scopes", - # attributes - ex_states=["h@pre"], - states=["h@alias"]) - - # create a stepnet for RNN - stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") - h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@alias") - - for op in [x_fc_op, h_fc_op, sum_op, sig_op]: - stepnet.append_op(op) - stepnet.complete_add_op(True) - self.forward_op.set_stepnet(stepnet) - - def create_gradient_op(self): - a = set() - backward_op = core.RecurrentOp.backward(self.forward_op, a) - - def test_grad(self): - self.create_forward_op() - self.create_gradient_op() + f[...] = o - delta + y_neg = self.forward() + + f[...] = o + dout_dfeed = (y_pos - y_neg) / (delta * 2) + g[...] = dout_dfeed[0] + + return grad_list + + +class RecurrentOpTest2(RecurrentOpTest1): + ''' + Test RNNOp + equation: + h_t = \sigma (W x_t + U h_{t-1}) + weights: + - W + - U + vars: + - x + memories: + - h + outputs: + - h + ''' + + input_dim = 2 + batch_size = 10 + sent_len = 2 + + def setUp(self): + self.setup_program() + + self.data_field = {"x", "h_boot", "W", "U"} + + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = PySimpleRNN2(self.input_shape, self.output_shape) + + self.output = mean(x=self.create_rnn_op(), **self.p_info) + + def create_rnn_op(self): + x = data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot = data( + shape=[self.input_dim], + data_type='float32', + name='h_boot', + **self.p_info) + h_boot.stop_gradient = False + + rnn = StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre = rnn.memory(init=h_boot) + x_t = rnn.step_input(x) + + temp_l = fc(input=x_t, + size=self.input_dim, + param_attr={'name': 'W'}, + bias_attr=False, + **self.p_info) + temp_r = fc(input=h_pre, + size=self.input_dim, + param_attr={'name': 'U'}, + bias_attr=False, + **self.p_info) + + h = sigmoid( + x=elementwise_add( + x=temp_l, y=temp_r, **self.p_info), + **self.p_info) + + rnn.update_memory(h_pre, h) + rnn.output(h) + + return rnn() + + +class RecurrentOpTest3(RecurrentOpTest1): + ''' + Test RNNOp with two memories + equation: + h_1 = h_pre_1 + h_2 = h_pre_2 + y = h_1 + h_2 + vars: + - x + memories: + - h_1, h_2 + outputs: + - y + ''' + + class PySimpleRNN3(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(RecurrentOpTest3.PySimpleRNN3, self).__init__(input_shape, + output_shape) + + seq_len, batch_size, input_dim = input_shape + self.h_boot1 = np.random.normal(size=(batch_size, + input_dim)).astype("float32") + self.h_boot2 = np.random.normal(size=(batch_size, + input_dim)).astype("float32") + + men_dim = (seq_len, batch_size, input_dim) + self.mems1 = np.zeros(shape=men_dim).astype("float32") + self.mems2 = np.zeros(shape=men_dim).astype("float32") + + def step(self, step_id, x): + if step_id == 0: + pre_mem1 = self.h_boot1 + pre_mem2 = self.h_boot2 + else: + pre_mem1 = self.mems1[step_id - 1] + pre_mem2 = self.mems2[step_id - 1] + self.mems1[step_id] = pre_mem1 + self.mems2[step_id] = pre_mem2 + self.y[step_id] = self.mems1[step_id] + self.mems2[step_id] + x + + input_dim = 1 + batch_size = 1 + sent_len = 2 + + def setUp(self): + self.setup_program() + + self.data_field = {"x", "h_boot1", "h_boot2"} + + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = RecurrentOpTest3.PySimpleRNN3(self.input_shape, + self.output_shape) + + self.output = mean(x=self.create_rnn_op(), **self.p_info) + + def create_rnn_op(self): + x = data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot1 = data( + shape=[self.batch_size, self.input_dim], + data_type='float32', + name='h_boot1', + append_batch_size=False, + **self.p_info) + h_boot1.stop_gradient = False + h_boot2 = data( + shape=[self.batch_size, self.input_dim], + data_type='float32', + name='h_boot2', + append_batch_size=False, + **self.p_info) + h_boot2.stop_gradient = False + + rnn = StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre1 = rnn.memory(init=h_boot1) + h_pre2 = rnn.memory(init=h_boot2) + x_t = rnn.step_input(x) + + mem1 = scale(x=h_pre1, scale=1.0, **self.p_info) + mem2 = scale(x=h_pre2, scale=1.0, **self.p_info) + out = sums(input=[mem1, x_t, mem2], **self.p_info) + + rnn.update_memory(h_pre1, mem1) + rnn.update_memory(h_pre2, mem2) + rnn.output(out) + + return rnn() if __name__ == '__main__': - exit( - 0 - ) # FIXME(qijun): https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957 unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rnn_helpers.py b/python/paddle/v2/framework/tests/test_rnn_helpers.py deleted file mode 100644 index be0ecfb129aa181229bc42d8d6818ad860991965..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_rnn_helpers.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest -from paddle.v2.framework.layers import * -from paddle.v2.framework.framework import g_program - - -class TestRNN(unittest.TestCase): - def test_rnn(self): - img = data( - shape=[ - 80, # sequence length - 22, # image height - 22 - ], # image width - data_type='float32', - name='image') - hidden = fc(input=img, size=100, act='sigmoid', num_flatten_dims=2) - self.assertEqual((-1, 80, 100), hidden.shape) - hidden = fc(input=hidden, size=100, act='sigmoid', num_flatten_dims=2) - self.assertEqual((-1, 80, 100), hidden.shape) - - rnn = StaticRNN() - with rnn.step(): - hidden = rnn.step_input(hidden) - self.assertEqual((-1, 100), hidden.shape) - memory = rnn.memory(shape=(-1, 32), dtype='float32', init_value=0.0) - - rnn_out = fc(input=[hidden, memory], size=32, act='sigmoid') - self.assertEqual((-1, 32), rnn_out.shape) - rnn.update_memory(memory, rnn_out) - rnn.output(rnn_out) - - out = rnn() - self.assertEqual((-1, 80, 32), out.shape) - print g_program - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py b/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py new file mode 100644 index 0000000000000000000000000000000000000000..731beff17cc96d26c2d9390a956c774b8676b179 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py @@ -0,0 +1,130 @@ +import unittest + +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +import numpy as np +import paddle.v2.framework.core as core + + +def create_tensor(np_data, place): + tensor = core.LoDTensor() + tensor.set(np_data, place) + return tensor + + +class RNNMemoryHelperOpTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.place = core.CPUPlace() + + self.X = self.program.global_block().create_var( + name='X', shape=[2, 3], dtype='float32') + self.Out = self.program.global_block().create_var( + name='Out', shape=[2, 3], dtype='float32') + self.program.global_block().append_op( + type='rnn_memory_helper', + inputs={"X": self.X}, + outputs={"Out": self.Out}, + attrs={}) + + def test_forward(self): + x_np = np.random.normal(size=(2, 3)).astype("float32") + self.feed_map = {'X': create_tensor(x_np, self.place)} + self.fetch_list = [self.Out] + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose(np.array(out[0]), x_np, rtol=1e-5) + + +class RNNMemoryHelperGradOpTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.place = core.CPUPlace() + + self.input_names = ['X', 'Out', 'Out@GRAD'] + self.input_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.input_names + } + + self.output_names = ['X@GRAD'] + self.output_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.output_names + } + + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}) + + def test_backward(self): + self.feed_map = { + name: create_tensor( + np.random.normal(size=(2, 3)).astype("float32"), self.place) + for name in self.input_names + } + self.fetch_list = [self.output_vars['X@GRAD']] + + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose(np.array(out[0]), self.feed_map['Out@GRAD'], rtol=1e-5) + + +class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.fake_program = Program() + self.place = core.CPUPlace() + + self.input_names = ['X', 'Out'] + self.input_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.input_names + } + self.input_vars["Out@GRAD"] = \ + self.fake_program.global_block().create_var( + name="Out@GRAD", shape=[2, 3], dtype='float32') + + self.output_names = ['X@GRAD'] + self.output_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.output_names + } + + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}) + + def test_backward(self): + self.feed_map = { + name: create_tensor( + np.random.normal(size=(2, 3)).astype("float32"), self.place) + for name in ['X', 'Out'] + } + self.fetch_list = [self.output_vars['X@GRAD']] + + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose( + np.array(out[0]), + np.zeros(shape=(2, 3)).astype("float32"), + rtol=1e-5) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_seq_conv.py b/python/paddle/v2/framework/tests/test_seq_conv.py index f0337c20a9e87fab971f9d9e2a113346feb20957..14edc5f953022ca05f5620c28bd7276d961dd4d0 100644 --- a/python/paddle/v2/framework/tests/test_seq_conv.py +++ b/python/paddle/v2/framework/tests/test_seq_conv.py @@ -45,10 +45,10 @@ class TestSeqProject(OpTest): self.inputs_val_no_f = ['PaddingData', 'X'] self.attrs = { - 'context_start': self.context_start, - 'context_length': self.context_length, - 'padding_trainable': self.padding_trainable, - 'context_stride': self.context_stride + 'contextStart': self.context_start, + 'contextLength': self.context_length, + 'paddingTrainable': self.padding_trainable, + 'contextStride': self.context_stride } out = np.zeros( (self.input_size[0], self.output_represention)).astype('float32') diff --git a/python/paddle/v2/framework/tests/test_seq_pool.py b/python/paddle/v2/framework/tests/test_seq_pool.py index 56602c57e6b63b71d6b089e774a876ad6164040e..efc4920124afb539017a3b3f211c7320da68ffef 100644 --- a/python/paddle/v2/framework/tests/test_seq_pool.py +++ b/python/paddle/v2/framework/tests/test_seq_pool.py @@ -3,15 +3,6 @@ import numpy as np from op_test import OpTest -class SeqPoolType(OpTest): - AVERAGE = 0 - SUM = 1 - SQRT = 2 - MAX = 3 - LAST = 4 - FIRST = 5 - - class TestSeqAvgPool(OpTest): def set_data(self): self.op_type = 'sequence_pool' @@ -25,7 +16,7 @@ class TestSeqAvgPool(OpTest): return x, lod, out def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.AVERAGE} + self.attrs = {'pooltype': "AVERAGE"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = sub_x.mean(axis=0) @@ -54,7 +45,7 @@ class TestSeqAvgPool2D(TestSeqAvgPool): return x, lod, out def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.AVERAGE} + self.attrs = {'pooltype': "AVERAGE"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) out[i] = np.reshape(sub_x.mean(axis=0), (3, 17)) @@ -62,7 +53,7 @@ class TestSeqAvgPool2D(TestSeqAvgPool): class TestSeqSumPool(TestSeqAvgPool): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.SUM} + self.attrs = {'pooltype': "SUM"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = sub_x.sum(axis=0) @@ -70,7 +61,7 @@ class TestSeqSumPool(TestSeqAvgPool): class TestSeqSumPool2D(TestSeqAvgPool2D): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.SUM} + self.attrs = {'pooltype': "SUM"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) out[i] = np.reshape(sub_x.sum(axis=0), (3, 17)) @@ -78,7 +69,7 @@ class TestSeqSumPool2D(TestSeqAvgPool2D): class TestSeqSqrtPool(TestSeqAvgPool): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.SQRT} + self.attrs = {'pooltype': "SQRT"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] len = lod[0][i + 1] - lod[0][i] @@ -87,7 +78,7 @@ class TestSeqSqrtPool(TestSeqAvgPool): class TestSeqSqrtPool2D(TestSeqAvgPool2D): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.SQRT} + self.attrs = {'pooltype': "SQRT"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) len = lod[0][i + 1] - lod[0][i] @@ -99,7 +90,7 @@ class TestSeqSqrtPool2D(TestSeqAvgPool2D): class TestSeqMaxPool(TestSeqAvgPool): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.MAX} + self.attrs = {'pooltype': "MAX"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = np.amax(sub_x, axis=0) @@ -111,7 +102,7 @@ class TestSeqMaxPool(TestSeqAvgPool): class TestSeqMaxPool2D(TestSeqAvgPool2D): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.MAX} + self.attrs = {'pooltype': "MAX"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 17)) @@ -123,7 +114,7 @@ class TestSeqMaxPool2D(TestSeqAvgPool2D): class TestSeqLastPool(TestSeqAvgPool): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.LAST} + self.attrs = {'pooltype': "LAST"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = sub_x[-1, :] @@ -131,7 +122,7 @@ class TestSeqLastPool(TestSeqAvgPool): class TestSeqLastPool2D(TestSeqAvgPool2D): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.LAST} + self.attrs = {'pooltype': "LAST"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) out[i] = np.reshape(sub_x[-1, :], (3, 17)) @@ -139,7 +130,7 @@ class TestSeqLastPool2D(TestSeqAvgPool2D): class TestSeqFirstPool(TestSeqAvgPool): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.FIRST} + self.attrs = {'pooltype': "FIRST"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = sub_x[0, :] @@ -147,7 +138,7 @@ class TestSeqFirstPool(TestSeqAvgPool): class TestSeqFirstPool2D(TestSeqAvgPool2D): def compute(self, x, lod, out): - self.attrs = {'strategy': SeqPoolType.FIRST} + self.attrs = {'pooltype': "FIRST"} for i in range(4): sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) out[i] = np.reshape(sub_x[0, :], (3, 17)) diff --git a/python/paddle/v2/framework/tests/test_top_k_op.py b/python/paddle/v2/framework/tests/test_top_k_op.py index 694f37d612d4c46e673dc894b05a0a446190732c..6e8fbefa6eafa391cdb5e17c882ee74b5bdc6507 100644 --- a/python/paddle/v2/framework/tests/test_top_k_op.py +++ b/python/paddle/v2/framework/tests/test_top_k_op.py @@ -9,7 +9,7 @@ class TestTopkOp(OpTest): k = 1 input = np.random.random((32, 84)).astype("float32") output = np.ndarray((32, k)) - indices = np.ndarray((32, k)) + indices = np.ndarray((32, k)).astype("int64") self.inputs = {'X': input} self.attrs = {'k': k} @@ -32,7 +32,7 @@ class TestTopkOp3d(OpTest): input = np.random.random((32, 2, 84)).astype("float32") input_flat_2d = input.reshape(64, 84) output = np.ndarray((64, k)) - indices = np.ndarray((64, k)).astype("int") + indices = np.ndarray((64, k)).astype("int64") # FIXME: should use 'X': input for a 3d input self.inputs = {'X': input_flat_2d} diff --git a/python/paddle/v2/framework/tests/test_understand_sentiment_conv.py b/python/paddle/v2/framework/tests/test_understand_sentiment_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..eb377e9264b6031e9bf484a90b7c2b39442407f1 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_understand_sentiment_conv.py @@ -0,0 +1,99 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program, g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def convolution_net(input_dim, class_dim=2, emb_dim=32, hid_dim=32): + data = layers.data(name="words", shape=[1], data_type="int64") + label = layers.data(name="label", shape=[1], data_type="int64") + + emb = layers.embedding(input=data, size=[input_dim, emb_dim]) + conv_3 = nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=3, + act="tanh", + pool_type="sqrt") + conv_4 = nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=4, + act="tanh", + pool_type="sqrt") + prediction = layers.fc(input=[conv_3, conv_4], + size=class_dim, + act="softmax") + cost = layers.cross_entropy(input=prediction, label=label) + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def main(): + BATCH_SIZE = 100 + PASS_NUM = 5 + + word_dict = paddle.dataset.imdb.word_dict() + dict_dim = len(word_dict) + class_dim = 2 + + cost, acc = convolution_net(input_dim=dict_dim, class_dim=class_dim) + + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000), + batch_size=BATCH_SIZE) + place = core.CPUPlace() + exe = Executor(place) + + exe.run(g_startup_program) + + for pass_id in xrange(PASS_NUM): + for data in train_data(): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([BATCH_SIZE, 1]) + + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if cost_val < 1.0 and acc_val > 0.7: + exit(0) + exit(1) + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/test_variable.py b/python/paddle/v2/framework/tests/test_variable.py index c670ca19afbd778747303cb002666aa2a5e62c37..03115f10a5a494424c6f8310c544c569be818e5b 100644 --- a/python/paddle/v2/framework/tests/test_variable.py +++ b/python/paddle/v2/framework/tests/test_variable.py @@ -1,5 +1,5 @@ import unittest -from paddle.v2.framework.framework import Variable, g_program, Program +from paddle.v2.framework.framework import Variable, g_main_program, Program import paddle.v2.framework.core as core import numpy as np @@ -18,7 +18,7 @@ class TestVariable(unittest.TestCase): self.assertRaises(ValueError, lambda: convert("int8")) def test_var(self): - b = g_program.current_block() + b = g_main_program.current_block() w = b.create_var( dtype="float64", shape=[784, 100], lod_level=0, name="fc.w") self.assertNotEqual(str(w), "") diff --git a/python/paddle/v2/framework/tests/test_word2vec.py b/python/paddle/v2/framework/tests/test_word2vec.py index 515d30d3e23edf429304d796faa8e17532168e26..6c3a448ec79717da6b81f51d9a4dccf038539b79 100644 --- a/python/paddle/v2/framework/tests/test_word2vec.py +++ b/python/paddle/v2/framework/tests/test_word2vec.py @@ -3,13 +3,13 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program, g_main_program from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() embed_size = 32 hidden_size = 256 @@ -24,32 +24,32 @@ first_word = layers.data( name='firstw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) second_word = layers.data( name='secondw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) third_word = layers.data( name='thirdw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) forth_word = layers.data( name='forthw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) next_word = layers.data( name='nextw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_first = layers.embedding( input=first_word, @@ -57,16 +57,16 @@ embed_first = layers.embedding( data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_second = layers.embedding( input=second_word, size=[dict_size, embed_size], data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_third = layers.embedding( input=third_word, @@ -74,42 +74,43 @@ embed_third = layers.embedding( data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_forth = layers.embedding( input=forth_word, size=[dict_size, embed_size], data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) hidden1 = layers.fc(input=concat_embed, size=hidden_size, act='sigmoid', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) predict_word = layers.fc(input=hidden1, size=dict_size, act='softmax', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( input=predict_word, label=next_word, - program=program, - init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +opts = sgd_optimizer.minimize(avg_cost, startup_program) train_reader = paddle.batch( paddle.dataset.imikolov.train(word_dict, N), batch_size) @@ -117,7 +118,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): @@ -145,7 +146,7 @@ for pass_id in range(PASS_NUM): next_tensor = core.LoDTensor() next_tensor.set(next_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={ 'firstw': first_tensor, 'secondw': second_tensor, diff --git a/python/paddle/v2/plot/plot.py b/python/paddle/v2/plot/plot.py index 6f7bd039b07db4832295c2374293bffa588eb4ef..c18e63dd5f60481ba804738a6a9238dfea35d9f3 100644 --- a/python/paddle/v2/plot/plot.py +++ b/python/paddle/v2/plot/plot.py @@ -56,7 +56,7 @@ class Ploter(object): assert isinstance(data, PlotData) data.append(step, value) - def plot(self): + def plot(self, path=None): if self.__plot_is_disabled__(): return @@ -68,8 +68,11 @@ class Ploter(object): titles.append(title) self.plt.plot(data.step, data.value) self.plt.legend(titles, loc='upper left') - self.display.clear_output(wait=True) - self.display.display(self.plt.gcf()) + if path is None: + self.display.clear_output(wait=True) + self.display.display(self.plt.gcf()) + else: + self.plt.savefig(path) self.plt.gcf().clear() def reset(self): diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index b68fd0d5a97a7993ddd0a1d947304fa5428c01b8..db01ab7374eca18b6063dc634da5ef83c4bc9adc 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -205,7 +205,8 @@ class SGD(object): """ Testing method. Will test input data. - :param reader: A reader that reads and yeilds data items. + :param reader: A batch reader that reads and yeilds data items, + it should be a paddle.v2.batch. :type reader: collections.Iterable :param feeding: Feeding is a map of neural network input name and array index that reader returns. diff --git a/python/requirements.txt b/python/requirements.txt index e19453c25da1ec78773c00a72b8e517b0d798fff..daf3f368b92408408897e33223118fe3647aa6de 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -7,3 +7,4 @@ rarfile scipy>=0.19.0 Pillow nltk>=3.2.2 +graphviz