diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b564b48265897d8b412603baf181030e2b00f82..4921226ec1c90a969fa1cfc383823820500c7757 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ endif() if(ANDROID OR IOS) if(ANDROID) - if(AND ${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") + if(${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") message(FATAL_ERROR "Unsupport standalone toolchains with Android API level lower than 16") elseif(${CMAKE_SYSTEM_VERSION} VERSION_LESS "21") # TODO: support glog for Android api 16 ~ 19 in the future diff --git a/cmake/generic.cmake b/cmake/generic.cmake index d2aab938d4636b1583062e27b73cb30f5d56b7b0..0bbf92293168d4e3af2c1ed0e82b75e6a8d6c0cd 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -106,22 +106,22 @@ function(merge_static_libs TARGET_NAME) endforeach() list(REMOVE_DUPLICATES libs_deps) - if(APPLE) # Use OSX's libtool to merge archives - # To produce a library we need at least one source file. - # It is created by add_custom_command below and will helps - # also help to track dependencies. - set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_dummy.c) + # To produce a library we need at least one source file. + # It is created by add_custom_command below and will helps + # also help to track dependencies. + set(target_SRCS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_dummy.c) + if(APPLE) # Use OSX's libtool to merge archives # Make the generated dummy source file depended on all static input # libs. If input lib changes,the source file is touched # which causes the desired effect (relink). - add_custom_command(OUTPUT ${dummyfile} - COMMAND ${CMAKE_COMMAND} -E touch ${dummyfile} + add_custom_command(OUTPUT ${target_SRCS} + COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} DEPENDS ${libs}) # Generate dummy staic lib - file(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") - add_library(${TARGET_NAME} STATIC ${dummyfile}) + file(WRITE ${target_SRCS} "const char *dummy = \"${target_SRCS}\";") + add_library(${TARGET_NAME} STATIC ${target_SRCS}) target_link_libraries(${TARGET_NAME} ${libs_deps}) foreach(lib ${libs}) @@ -130,11 +130,14 @@ function(merge_static_libs TARGET_NAME) endforeach() add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" - COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles}) + COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles} + ) else() # general UNIX: use "ar" to extract objects and re-add to a common lib + set(target_DIR ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.dir) + foreach(lib ${libs}) - set(objlistfile ${lib}.objlist) # list of objects in the input library - set(objdir ${lib}.objdir) + set(objlistfile ${target_DIR}/${lib}.objlist) # list of objects in the input library + set(objdir ${target_DIR}/${lib}.objdir) add_custom_command(OUTPUT ${objdir} COMMAND ${CMAKE_COMMAND} -E make_directory ${objdir} @@ -142,31 +145,32 @@ function(merge_static_libs TARGET_NAME) add_custom_command(OUTPUT ${objlistfile} COMMAND ${CMAKE_AR} -x "$" - COMMAND ${CMAKE_AR} -t "$" > ../${objlistfile} + COMMAND ${CMAKE_AR} -t "$" > ${objlistfile} DEPENDS ${lib} ${objdir} WORKING_DIRECTORY ${objdir}) - # Empty dummy source file that goes into merged library - set(mergebase ${lib}.mergebase.c) - add_custom_command(OUTPUT ${mergebase} - COMMAND ${CMAKE_COMMAND} -E touch ${mergebase} - DEPENDS ${objlistfile}) - - list(APPEND mergebases "${mergebase}") + list(APPEND target_OBJS "${objlistfile}") endforeach() - add_library(${TARGET_NAME} STATIC ${mergebases}) + # Make the generated dummy source file depended on all static input + # libs. If input lib changes,the source file is touched + # which causes the desired effect (relink). + add_custom_command(OUTPUT ${target_SRCS} + COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} + DEPENDS ${libs} ${target_OBJS}) + + # Generate dummy staic lib + file(WRITE ${target_SRCS} "const char *dummy = \"${target_SRCS}\";") + add_library(${TARGET_NAME} STATIC ${target_SRCS}) target_link_libraries(${TARGET_NAME} ${libs_deps}) # Get the file name of the generated library - set(outlibfile "$") + set(target_LIBNAME "$") - foreach(lib ${libs}) - add_custom_command(TARGET ${TARGET_NAME} POST_BUILD - COMMAND ${CMAKE_AR} cr ${outlibfile} *.o - COMMAND ${CMAKE_RANLIB} ${outlibfile} - WORKING_DIRECTORY ${lib}.objdir) - endforeach() + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_AR} crs ${target_LIBNAME} `find ${target_DIR} -name '*.o'` + COMMAND ${CMAKE_RANLIB} ${target_LIBNAME} + WORKING_DIRECTORY ${target_DIR}) endif() endfunction(merge_static_libs) @@ -196,7 +200,7 @@ function(cc_library TARGET_NAME) add_style_check_target(${TARGET_NAME} ${cc_library_SRCS} ${cc_library_HEADERS}) else(cc_library_SRCS) - if (cc_library_DEPS) + if(cc_library_DEPS) merge_static_libs(${TARGET_NAME} ${cc_library_DEPS}) else() message(FATAL "Please specify source file or library in cc_library.") diff --git a/cmake/util.cmake b/cmake/util.cmake index e814cad36f2a8ce95a2dc9fabc35cb39506d4cd7..ac911052eb970c5a3e485e3178dd788b1517ca30 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -25,7 +25,7 @@ function(target_circle_link_libraries TARGET_NAME) endif() endforeach() if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - if(IOS AND NOT IOS_ENABLE_BITCODE) + if(NOT IOS_ENABLE_BITCODE) list(APPEND LIBS "-undefined dynamic_lookup") endif() endif() diff --git a/doc/design/api.md b/doc/design/api.md index 8185d2af0ea264a2e7b4e28b9ed05279e4a22014..e6a4638d9100d9b07c3ee6b92b530a17eae1c162 100644 --- a/doc/design/api.md +++ b/doc/design/api.md @@ -3,7 +3,7 @@ ## Ingredients As our design principle is starting from the essence: how could we -allow users to express and solve their problems at neural networks. +allow users to express and solve their problems as neural networks. Some essential concepts that our API have to provide include: 1. A *topology* is an expression of *layers*. @@ -233,7 +233,7 @@ paddle.dist_train(model, num_parameter_servers=15) ``` -The pseudo code if `paddle.dist_train` is as follows: +The pseudo code of `paddle.dist_train` is as follows: ```python def dist_train(topology, parameters, trainer, reader, ...): diff --git a/doc/design/auto_gradient_check.md b/doc/design/auto_gradient_check.md index 1f4d4ec16f7c395005e610751d95c10f5f3adf52..f9991541bc51c6e13ffce4e9cec60f73dc800121 100644 --- a/doc/design/auto_gradient_check.md +++ b/doc/design/auto_gradient_check.md @@ -1,17 +1,17 @@ ## Auto Gradient Checker Design ## Backgraound： -- Operator forward computing is easy to check if the result is right because it has a clear definition. **But** backpropagation is a notoriously difficult algorithm to debug and get right: - - 1. you should get the right backpropagation formula according to the forward computation. - - 2. you should implement it right in CPP. - - 3. it's difficult to prepare test data. +- Generally, it is easy to check whether the forward computation of an Operator is correct or not. However, backpropagation is a notoriously difficult algorithm to debug and get right: + 1. you should get the right backpropagation formula according to the forward computation. + 2. you should implement it right in CPP. + 3. it's difficult to prepare test data. -- Auto gradient check gets a numeric gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: - - 1. numeric gradient checker only need forward operator. - - 2. user only need to prepare the input data for forward Operator. +- Auto gradient checking gets a numerical gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: + 1. numerical gradient checker only need forward operator. + 2. user only need to prepare the input data for forward Operator. ## Mathematical Theory -The following two document from stanford has a detailed explanation of how to get numeric gradient and why it's useful. +The following two document from Stanford has a detailed explanation of how to get numerical gradient and why it's useful. - [Gradient checking and advanced optimization(en)](http://deeplearning.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization) - [Gradient checking and advanced optimization(cn)](http://ufldl.stanford.edu/wiki/index.php/%E6%A2%AF%E5%BA%A6%E6%A3%80%E9%AA%8C%E4%B8%8E%E9%AB%98%E7%BA%A7%E4%BC%98%E5%8C%96) @@ -20,7 +20,7 @@ The following two document from stanford has a detailed explanation of how to ge ## Numeric Gradient Implementation ### Python Interface ```python -def get_numeric_gradient(op, +def get_numerical_gradient(op, input_values, output_name, input_to_check, @@ -30,13 +30,13 @@ def get_numeric_gradient(op, Get Numeric Gradient for an operator's input. :param op: C++ operator instance, could be an network - :param input_values: The input variables. Should be an dictionary, key is - variable name. Value is numpy array. + :param input_values: The input variables. Should be an dictionary, whose key is + variable name, and value is numpy array. :param output_name: The final output variable name. - :param input_to_check: The input variable need to get gradient. + :param input_to_check: The input variable with respect to which to compute the gradient. :param delta: The perturbation value for numeric gradient method. The smaller delta is, the more accurate result will get. But if that delta is - too small, it could occur numerical stability problem. + too small, it will suffer from numerical stability problem. :param local_scope: The local scope used for get_numeric_gradient. :return: The gradient array in numpy format. """ @@ -45,28 +45,28 @@ def get_numeric_gradient(op, ### Explaination: - Why need `output_name` - - One Operator may have multiple Output, you can get independent gradient from each Output. So user should set one output to calculate. + - An Operator may have multiple Output, one can get independent gradient from each Output. So caller should specify the name of the output variable. - Why need `input_to_check` - - One operator may have multiple inputs. Gradient Op can calculate the gradient of these Inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. + - One operator may have multiple inputs. Gradient Op can calculate the gradient of these inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. ### Core Algorithm Implementation ```python - # we only compute gradient of one element each time. - # we use a for loop to compute the gradient of every element. + # we only compute gradient of one element a time. + # we use a for loop to compute the gradient of each element. for i in xrange(tensor_size): - # get one input element throw it's index i. + # get one input element by its index i. origin = tensor_to_check.get_float_element(i) - # add delta to it, run op and then get the sum of the result tensor. + # add delta to it, run op and then get the new value of the result tensor. x_pos = origin + delta tensor_to_check.set_float_element(i, x_pos) y_pos = get_output() - # plus delta to this element, run op and get the sum of the result tensor. + # plus delta to this element, run op and get the new value of the result tensor. x_neg = origin - delta tensor_to_check.set_float_element(i, x_neg) y_neg = get_output() @@ -85,15 +85,15 @@ def get_numeric_gradient(op, Each Operator Kernel has three kinds of Gradient: -- 1. Numeric Gradient -- 2. CPU Operator Gradient -- 3. GPU Operator Gradient(if supported) +1. Numerical gradient +2. CPU kernel gradient +3. GPU kernel gradient (if supported) -Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as the reference value. +The numerical gradient only relies on forward Operator. So we use the numerical gradient as the reference value. And the gradient checking is performed in the following three steps: -- 1. calculate the numeric gradient. -- 2. calculate CPU kernel Gradient with the backward Operator and compare it with the numeric gradient. -- 3. calculate GPU kernel Gradient with the backward Operator and compare it with the numeric gradient.(if support GPU) +1. calculate the numerical gradient +2. calculate CPU kernel gradient with the backward Operator and compare it with the numerical gradient +3. calculate GPU kernel gradient with the backward Operator and compare it with the numeric gradient (if supported) #### Python Interface @@ -110,8 +110,8 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as :param forward_op: used to create backward_op :param input_vars: numpy value of input variable. The following computation will use these variables. - :param inputs_to_check: inputs var names that should check gradient. - :param output_name: output name that used to + :param inputs_to_check: the input variable with respect to which to compute the gradient. + :param output_name: The final output variable name. :param max_relative_error: The relative tolerance parameter. :param no_grad_set: used when create backward ops :param only_cpu: only compute and check gradient on cpu kernel. @@ -120,24 +120,24 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as ``` ### How to check if two numpy array is close enough? -if `abs_numeric_grad` is nearly zero, then use abs error for numeric_grad, not relative +if `abs_numerical_grad` is nearly zero, then use abs error for numerical_grad ```python -numeric_grad = ... +numerical_grad = ... operator_grad = numpy.array(scope.find_var(grad_var_name(name)).get_tensor()) -abs_numeric_grad = numpy.abs(numeric_grad) -# if abs_numeric_grad is nearly zero, then use abs error for numeric_grad, not relative +abs_numerical_grad = numpy.abs(numerical_grad) +# if abs_numerical_grad is nearly zero, then use abs error for numeric_grad, not relative # error. -abs_numeric_grad[abs_numeric_grad < 1e-3] = 1 +abs_numerical_grad[abs_numerical_grad < 1e-3] = 1 -diff_mat = numpy.abs(abs_numeric_grad - operator_grad) / abs_numeric_grad +diff_mat = numpy.abs(abs_numerical_grad - operator_grad) / abs_numerical_grad max_diff = numpy.max(diff_mat) ``` #### Notes： -1，The Input data for auto gradient checker should be reasonable to avoid numeric problem. +The Input data for auto gradient checker should be reasonable to avoid numerical stability problem. #### Refs: diff --git a/doc/design/functions_operators_layers.md b/doc/design/functions_operators_layers.md index d23ba56b5773a36d448a99e4abdebc1475ed789c..984b59f4c6971dfb6f46dfe342f2751f392c0e88 100644 --- a/doc/design/functions_operators_layers.md +++ b/doc/design/functions_operators_layers.md @@ -53,12 +53,12 @@ Let's explain using an example. Suppose that we are going to compose the FC usi ```python def operator.mul(X1, X2): O = Var() - paddle.cpp.create_operator("mul", input={X1, Y1], output=O) + paddle.cpp.create_operator("mul", input={X1, Y1}, output=O) return O def operator.add(X1, X2): O = Var() - paddle.cpp.create_operator("add", input={X1, X2], output=O) + paddle.cpp.create_operator("add", input={X1, X2}, output=O) return O ``` diff --git a/doc/design/graph.md b/doc/design/graph.md index 51b7f87638f8ddff752328a562fe0dd0fe56cfd1..7519a65df835a39fe14f6ef45530afff170191ff 100644 --- a/doc/design/graph.md +++ b/doc/design/graph.md @@ -56,7 +56,7 @@ For each parameter, like W and b created by `layer.fc`, marked as double circles ## Block and Graph -The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block[(https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. +The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block](https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. A Block keeps operators in an array `BlockDesc::ops` @@ -67,4 +67,4 @@ message BlockDesc { } ``` -in the order that there appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. +in the order that they appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. diff --git a/doc/design/parameters_in_cpp.md b/doc/design/parameters_in_cpp.md index b6f99bc7d9d6fafacb0a4bcff806b65d9aef98cc..a7ac3f17c44ca94a669a8f1e283b291bceb42317 100644 --- a/doc/design/parameters_in_cpp.md +++ b/doc/design/parameters_in_cpp.md @@ -1,19 +1,19 @@ # Design Doc: The C++ Class `Parameters` -`Parameters` is a concept we designed in Paddle V2 API. `Parameters` is a container of parameters, and make Paddle can shared parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). +`Parameters` is a concept we designed in PaddlePaddle V2 API. `Parameters` is a container of parameters, which makes PaddlePaddle capable of sharing parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). -We used Python to implement Parameters when designing V2 API before. There are several defects for current implementation: +We used Python to implement Parameters when designing V2 API before. There are several defects for the current implementation: * We just use `memcpy` to share Parameters between topologies, but this is very inefficient. -* We did not implement share Parameters while training. We just trigger `memcpy` when start training. +* We did not support sharing Parameters while training. We just trigger `memcpy` when start training. -It is necessary that we implement Parameters in CPP side. However, it could be a code refactoring for Paddle, because Paddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current Paddle implementation, there are three concepts associated with `Parameters`: +It is necessary that we implement Parameters in CPP side. However, it could result a code refactoring for PaddlePaddle, because PaddlePaddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current PaddlePaddle implementation, there are three concepts associated with `Parameters`: 1. `paddle::Parameter`. A `Parameters` is a container for `paddle::Parameter`. It is evident that we should use `paddle::Parameter` when developing `Parameters`. However, the `Parameter` class contains many functions and does not have a clear interface. It contains `create/store Parameter`, `serialize/deserialize`, `optimize(i.e SGD)`, `randomize/zero`. When we developing `Parameters`, we only use `create/store Parameter` functionality. -We should extract functionalities of Parameter into many classes to clean Paddle CPP implementation. +We should extract functionalities of Parameter into many classes to clean PaddlePaddle CPP implementation. 2. `paddle::GradientMachine` and its sub-classes, e.g., `paddle::MultiGradientMachine`, `paddle::NeuralNetwork`. We should pass `Parameters` to `paddle::GradientMachine` when `forward/backward` to avoid `memcpy` between topologies. @@ -24,7 +24,7 @@ Also, we should handle multi-GPU/CPU training, because `forward` and `backward` So `Parameters` should be used by `paddle::ParameterUpdater`, and `paddle::ParameterUpdater` should optimize `Parameters` (by SGD). -The step by step approach for implementation Parameters in Paddle C++ core is listed below. Each step should be a PR and could be merged into Paddle one by one. +The step by step approach for implementation Parameters in PaddlePaddle C++ core is listed below. Each step should be a PR and could be merged into PaddlePaddle one by one. 1. Clean `paddle::Parameter` interface. Extract the functionalities of `paddle::Parameter` to prepare for the implementation of Parameters. diff --git a/doc/design/program.md b/doc/design/program.md new file mode 100644 index 0000000000000000000000000000000000000000..fb8f86ac07af403c9fee015f2a3adbfaa3c6d631 --- /dev/null +++ b/doc/design/program.md @@ -0,0 +1,61 @@ +# Design Doc: ProgramDesc + +The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program. + +As described in [graph.md](./graph.md), the first five lines of the following PaddlePaddle program + +```python +x = layer.data("images") +l = layer.data("label") +y = layer.fc(x) +cost = layer.mse(y, l) +optimize(cost) +train(cost, reader=mnist.train()) +``` + +generates, or compiles, a PaddelPaddle program, which is represented by the following protobuf message: + +```protobuf +message ProgramDesc { + repeated BlockDesc blocks = 1; +} + +message BlockDesc { + required int32 parent = 1; + repeated VarDesc vars = 2; + repeated OpDesc ops = 3; +} + +message OpDesc { + AttrDesc attrs = 1; + ... +} + +message AttrDesc { + required AttrType type = 1; + + // index into ProgramDesc::blocks when type==BLOCK + optional int32 block = 2; + ... +} +``` + +When each of the first five lines runs, related Python function, e.g., `layer.fc`, calls C++ InferShape functions. This InferShape function needs to access the properties of VarDesc's accessed by the current OpDesc. These VarDesc's might not be defined in the current block, but in some ancestor blocks. This requires that we can trace the parent of a block. + +A nested block is often an attribute of an operator, most likely, an IfElseOp or a WhileOp. In above solution, all blocks are in `ProgramDesc::blocks`, this implicitly assigns a zero-based ID to each block -- the index of the block in `ProgramDesc::blocks`. So that `AttrDesc::block` could be an integer block ID. + +With this design, the InferShape function should take the following parameters: + +```c++ +void InferShape(int current_block, + int current_operator, + ProgramDesc* program // might change VarDesc values. + ) { + ... +} +``` + +where + +- `current_block` indices into `ProgramDesc::blocks`, +- `current_operator` indices into `BlockDesc::ops`. diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index f21f7af520df5171798326818ecb97c3bcd14a12..320dccec3ddc7bfe6042f4e65b2518ea7b1ad24a 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -52,7 +52,7 @@ Here are valid outputs: # a mini batch of three data items, each data item is a list (single column). [([1,1,1],), ([2,2,2],), -([3,3,3],), +([3,3,3],)] ``` Please note that each item inside the list must be a tuple, below is an invalid output: diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md index e105861e926411a269b0b52dd4688744912c9ab3..ad801ca421ca31c84b0a6b0a18d1d625c87e0de5 100644 --- a/doc/design/refactorization.md +++ b/doc/design/refactorization.md @@ -15,7 +15,7 @@ The goal of refactorizaiton include: 1. Users write Python programs to describe the graphs and run it (locally or remotely). -1. A graph is composed of *variabels* and *operators*. +1. A graph is composed of *variables* and *operators*. 1. The description of graphs must be able to be serialized/deserialized, so it @@ -140,7 +140,7 @@ Compile Time -> IR -> Runtime * `thrust` has the same API as C++ standard library. Using `transform` can quickly implement a customized elementwise kernel. * `thrust` has more complex API, like `scan`, `reduce`, `reduce_by_key`. * Hand-writing `GPUKernel` and `CPU` code - * Do not write `.h`. CPU Kernel should be in `.cc`. CPU kernel should be in `.cu`. (`GCC` cannot compile GPU code.) + * Do not write `.h`. CPU Kernel should be in `.cc`. GPU kernel should be in `.cu`. (`GCC` cannot compile GPU code.) --- # Operator Register diff --git a/doc/design/releasing_process.md b/doc/design/releasing_process.md index 0c10e782808ca6456347ec54cb5e921162731ede..62ff8f3229bbbb5bc82e4da29259baffc30c2c87 100644 --- a/doc/design/releasing_process.md +++ b/doc/design/releasing_process.md @@ -1,8 +1,8 @@ -# Paddle发行规范 +# PaddlePaddle发行规范 -Paddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示Paddle版本号。 +PaddlePaddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示PaddlePaddle版本号。 -Paddle每次发新的版本，遵循以下流程: +PaddlePaddle每次发新的版本，遵循以下流程: 1. 从`develop`分支派生出新的分支，分支名为`release/版本号`。例如，`release/0.10.0` 2. 将新分支的版本打上tag，tag为`版本号rc.Patch号`。第一个tag为`0.10.0rc1`，第二个为`0.10.0rc2`，依次类推。 @@ -27,14 +27,14 @@ Paddle每次发新的版本，遵循以下流程: 需要注意的是: -* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试Paddle的行为。 +* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试PaddlePaddle的行为。 * 在`release/版本号`分支存在的时候，如果有bugfix的行为，需要将bugfix的分支同时merge到`master`, `develop`和`release/版本号`这三个分支。 -# Paddle 分支规范 +# PaddlePaddle 分支规范 -Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 +PaddlePaddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 -* Paddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: +* PaddlePaddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: * `master`分支为稳定(stable branch)版本分支。每一个`master`分支的版本都是经过单元测试和回归测试的版本。 * `develop`分支为开发(develop branch)版本分支。每一个`develop`分支的版本都经过单元测试，但并没有经过回归测试。 * `release/版本号`分支为每一次Release时建立的临时分支。在这个阶段的代码正在经历回归测试。 @@ -42,18 +42,18 @@ Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branch * 其他用户的fork版本库并不需要严格遵守[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，但所有fork的版本库的所有分支都相当于特性分支。 * 建议，开发者fork的版本库使用`develop`分支同步主版本库的`develop`分支 * 建议，开发者fork的版本库中，再基于`develop`版本fork出自己的功能分支。 - * 当功能分支开发完毕后，向Paddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 + * 当功能分支开发完毕后，向PaddlePaddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 * 在评审过程中，开发者修改自己的代码，可以继续在自己的功能分支提交代码。 * BugFix分支也是在开发者自己的fork版本库维护，与功能分支不同的是，BugFix分支需要分别给主版本库的`master`、`develop`与可能有的`release/版本号`分支，同时提起`Pull Request`。 -# Paddle回归测试列表 +# PaddlePaddle回归测试列表 -本列表说明Paddle发版之前需要测试的功能点。 +本列表说明PaddlePaddle发版之前需要测试的功能点。 -## Paddle Book中所有章节 +## PaddlePaddle Book中所有章节 -Paddle每次发版本首先要保证Paddle Book中所有章节功能的正确性。功能的正确性包括验证Paddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 +PaddlePaddle每次发版本首先要保证PaddlePaddle Book中所有章节功能的正确性。功能的正确性包括验证PaddlePaddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 | | 新手入门章节 | 识别数字 | 图像分类 | 词向量 | 情感分析 | 语意角色标注 | 机器翻译 | 个性化推荐 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | diff --git a/doc/design/scope.md b/doc/design/scope.md index c9e0be716b606f6c7bf0373e0c6e632647e07a6f..b1f9bb4378eb5ec6926f1e53f7c1f4fd5674064c 100644 --- a/doc/design/scope.md +++ b/doc/design/scope.md @@ -17,7 +17,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope only contains a map of a name to variable. - All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state(momentum) etc. + All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state (momentum) etc. 1. Variable can only be created by Scope and a variable can only be got from Scope. User cannot create or get a variable outside a scope. This is a constraints of our framework, and will keep our framework simple and clear. @@ -32,7 +32,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope should destruct all Variables inside it when itself is destructed. User can never store `Variable` pointer somewhere else. - Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be a invalid pointer when associated `Scope` is destroyed. + Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be an invalid pointer when associated `Scope` is destroyed. ```cpp class Scope { @@ -50,7 +50,7 @@ class Scope { Just like [scope](https://en.wikipedia.org/wiki/Scope_(computer_science)) in programming languages, `Scope` in the neural network can also be a local scope. There are two attributes about local scope. -1. We can create local variables in a local scope. When that local scope are destroyed, all local variables should also be destroyed. +1. We can create local variables in a local scope. When that local scope is destroyed, all local variables should also be destroyed. 2. Variables in a parent scope can be retrieved from local scopes of that parent scope, i.e., when user get a variable from a scope, it will try to search this variable in current scope. If there is no such variable in the local scope, `scope` will keep searching from its parent, until the variable is found or there is no parent. ```cpp @@ -121,4 +121,4 @@ Also, as the parent scope is a `shared_ptr`, we can only `Create()` a scope shar ## Orthogonal interface -`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return a `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. +`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return an `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. diff --git a/doc/design/simple_op_design.md b/doc/design/simple_op_design.md index fded4a68612396a262121a5a886a8ae573dfa662..c7aeed7f9b4637e1c29d530f37b42d12500af82f 100644 --- a/doc/design/simple_op_design.md +++ b/doc/design/simple_op_design.md @@ -6,9 +6,9 @@ The Interaction between Python and C++ can be simplified as two steps: 1. C++ tells Python how many Ops there are, and what parameter do users need to offer to initialize a new Op. Python then builds API for each Op at compile time. -2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ fo finish Op construction task. +2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ for finishing the Op construction task. -### Message form C++ to Python +### Message from C++ to Python We define a Protobuf message class `OpProto` to hold message needed in the first step. What should an `OpProto` contain? This question is equivalent to “What message do we need to offer, to build a Python API which is legal and user oriented and can use to describe a whole Op.” @@ -193,7 +193,7 @@ def fc_layer(input, size, with_bias, activation): elif: # ... return act_output; -``` +``` ### Low Leval API diff --git a/doc/design/var_desc.md b/doc/design/var_desc.md index 86a95c10d5729704f86c285c9fe92db0cf2158be..bfbbdd0578ebc69ea4b49ade9b041573a9e9ad55 100644 --- a/doc/design/var_desc.md +++ b/doc/design/var_desc.md @@ -1,7 +1,7 @@ ## Background PaddlePaddle divides the description of neural network computation graph into two stages: compile time and runtime. -PaddlePaddle use proto message to describe compile time graph for +PaddlePaddle use proto message to describe compile time graph because 1. Computation graph should be able to be saved to a file. 1. In distributed training, the graph will be serialized and send to multiple workers. diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 00192aa69bd487787a8743d5589a365eacbd4ff3..acbf4c87ae5242f6cfc593a7fddc649ee3a70d7c 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -158,17 +158,23 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 -7. \*-cp27mu-linux_x86_64.whl is not a supported wheel on this platform. +7. paddlepaddle\*.whl is not a supported wheel on this platform. ------------------------------------------------------------------------ -出现这个问题的主要原因是，系统编译wheel包的时候，使用的 :code:`wheel` 包是最新的， -而系统中的 :code:`pip` 包比较老。具体的解决方法是，更新 :code:`pip` 包并重新编译PaddlePaddle。 +出现这个问题的主要原因是，没有找到和当前系统匹配的paddlepaddle安装包。最新的paddlepaddle python安装包支持Linux x86_64和MacOS 10.12操作系统，并安装了python 2.7和pip 9.0.1。 + 更新 :code:`pip` 包的方法是\: .. code-block:: bash pip install --upgrade pip +如果还不行，可以执行 :code:`python -c "import pip; print(pip.pep425tags.get_supported())"` 获取当前系统支持的python包的后缀， +并对比是否和正在安装的后缀一致。 + +如果系统支持的是 :code:`linux_x86_64` 而安装包是 :code:`manylinux1_x86_64` ，需要升级pip版本到最新； +如果系统支持 :code:`manylinux1_x86_64` 而安装包（本地）是 :code:`linux_x86_64` ，可以重命名这个whl包为 :code:`manylinux1_x86_64` 再安装。 + 8. python相关的单元测试都过不了 -------------------------------- @@ -310,7 +316,7 @@ Paddle二进制在运行时捕获了浮点数异常，只要出现浮点数异 * 模型一直不收敛，发散到了一个数值特别大的地方。 * 训练数据有问题，导致参数收敛到了一些奇异的情况。或者输入数据尺度过大，有些特征的取值达到数百万，这时进行矩阵乘法运算就可能导致浮点数溢出。 -主要的解决办法是减小学习律或者对数据进行归一化处理。 +主要的解决办法是减小学习率或者对数据进行归一化处理。 15. 编译安装后执行 import paddle.v2 as paddle 报ImportError: No module named v2 ------------------------------------------------------------------------ @@ -373,3 +379,15 @@ PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数 parameters = paddle.parameters.create(my_cost) parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) + +18. 集群多节点训练，日志中保存均为网络通信类错误 +------------------------------ + +集群多节点训练，日志报错为网络通信类错误，比如 :code:`Connection reset by peer` 等。 +此类报错通常是由于某一个节点的错误导致这个节点的训练进程退出，从而引发其他节点无法连接导致，可以参考下面的步骤排查： + +* 从 :code:`train.log` ， :code:`server.log` 找到最早报错的地方，查看是否是其他错误引发的报错（比如FPE，内存不足，磁盘空间不足等）。 + +* 如果发现最早的报错就是网络通信的问题，很有可能是非独占方式执行导致的端口冲突，可以联系OP，看当前MPI集群是否支持resource=full参数提交，如果支持增加此参数提交，并更换job 端口。 + +* 如果当前MPI集群并不支持任务独占模式，可以联系OP是否可以更换集群或升级当前集群。 \ No newline at end of file diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index c6570b89aedfaac1aef9b00e889b0b3ed21d8d65..264b998f50df016da0741d97d4b26f759ee90900 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -54,9 +54,9 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { public: MulOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of mul op"); - AddInput("Y", "The second input of mul op"); - AddOutput("Out", "The output of mul op"); + AddInput("X", "(Tensor), 2D tensor of size (M x K)"); + AddInput("Y", "(Tensor), 2D tensor of size (K x N)"); + AddOutput("Out", "(Tensor), 2D tensor of size (M x N)"); AddComment(R"DOC( Two Element Mul Operator. The equation is: Out = X * Y @@ -72,7 +72,7 @@ The equation is: Out = X * Y 构造函数里通过`AddInput`添加输入参数，通过`AddOutput`添加输出参数，通过`AddComment`添加Op的注释。这些函数会将对应内容添加到`OpProto`中。 -上面的代码在`MulOp`中添加两个输入`X`和`Y`，添加了一个输出`Out`，并解释了各自含义，命名请遵守命名规范。 +上面的代码在`MulOp`中添加两个输入`X`和`Y`，添加了一个输出`Out`，并解释了各自含义，命名请遵守[命名规范](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/name_convention.md)。 再以[`ScaleOp`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37)为例： diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt index dd9e4f1cbd636e29a6934d1119fc93ebc9d0ecee..b9bbe58951c643f1b1649858880fbd2ba3a2a7b7 100644 --- a/paddle/capi/CMakeLists.txt +++ b/paddle/capi/CMakeLists.txt @@ -62,6 +62,7 @@ if(ANDROID) LIBRARY DESTINATION lib/${ANDROID_ABI}) execute_process( COMMAND ${GIT_EXECUTABLE} log --pretty=oneline -1 + WORKING_DIRECTORY ${PADDLE_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITS_LIST RESULT_VARIABLE GIT_COMMITS_LIST_RESULT ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -81,8 +82,7 @@ if(ANDROID) )" ) else(ANDROID) - install(TARGETS paddle_capi_whole - ARCHIVE DESTINATION lib) + install(TARGETS paddle_capi_whole ARCHIVE DESTINATION lib) if(NOT IOS) install(TARGETS paddle_capi_shared DESTINATION lib) endif() diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 3371962c635c3731f00a6af2a6e287ece33397cd..e535f84dba7c2726fbb70fa11ca8e9e2d29b8665 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -19,12 +19,14 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) proto_library(framework_proto SRCS framework.proto) cc_library(attribute SRCS attribute.cc DEPS framework_proto) +cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) +cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op) diff --git a/paddle/framework/attribute.cc b/paddle/framework/attribute.cc index 27132eaa0b3b0666fc042faf052dac2e169ba9e7..fda89252e35c382468877e8cab148e5f91d77ac2 100644 --- a/paddle/framework/attribute.cc +++ b/paddle/framework/attribute.cc @@ -19,6 +19,19 @@ limitations under the License. */ namespace paddle { namespace framework { +static ProgramDesc* g_program_desc = nullptr; + +ProgramDesc& GetProgramDesc() { + if (g_program_desc == nullptr) { + g_program_desc = new ProgramDesc(); + } + return *g_program_desc; +} + +template <> +AttrType AttrTypeID() { + return BOOLEAN; +} template <> AttrType AttrTypeID() { return INT; @@ -32,6 +45,10 @@ AttrType AttrTypeID() { return STRING; } template <> +AttrType AttrTypeID>() { + return BOOLEANS; +} +template <> AttrType AttrTypeID>() { return INTS; } @@ -47,40 +64,54 @@ template <> AttrType AttrTypeID>>() { return INT_PAIRS; } +template <> +AttrType AttrTypeID() { + return BLOCK; +} Attribute GetAttrValue(const OpDesc::Attr& attr_desc) { switch (attr_desc.type()) { - case paddle::framework::AttrType::INT: { + case framework::AttrType::BOOLEAN: { + return attr_desc.b(); + } + case framework::AttrType::INT: { return attr_desc.i(); } - case paddle::framework::AttrType::FLOAT: { + case framework::AttrType::FLOAT: { return attr_desc.f(); } - case paddle::framework::AttrType::STRING: { + case framework::AttrType::STRING: { return attr_desc.s(); } - case paddle::framework::AttrType::INTS: { + case framework::AttrType::BOOLEANS: { + std::vector val(attr_desc.bools_size()); + for (int i = 0; i < attr_desc.bools_size(); ++i) { + val[i] = attr_desc.bools(i); + } + return val; + } + case framework::AttrType::INTS: { std::vector val(attr_desc.ints_size()); for (int i = 0; i < attr_desc.ints_size(); ++i) { val[i] = attr_desc.ints(i); } return val; } - case paddle::framework::AttrType::FLOATS: { + case framework::AttrType::FLOATS: { std::vector val(attr_desc.floats_size()); for (int i = 0; i < attr_desc.floats_size(); ++i) { val[i] = attr_desc.floats(i); } return val; } - case paddle::framework::AttrType::STRINGS: { + case framework::AttrType::STRINGS: { std::vector val(attr_desc.strings_size()); for (int i = 0; i < attr_desc.strings_size(); ++i) { val[i] = attr_desc.strings(i); } return val; } - case paddle::framework::AttrType::INT_PAIRS: { + case framework::AttrType::INT_PAIRS: { std::vector> val(attr_desc.int_pairs_size()); for (int i = 0; i < attr_desc.int_pairs_size(); ++i) { val[i].first = attr_desc.int_pairs(i).first(); @@ -88,6 +119,9 @@ Attribute GetAttrValue(const OpDesc::Attr& attr_desc) { } return val; } + case framework::AttrType::BLOCK: { + return GetProgramDesc().mutable_blocks(attr_desc.block_idx()); + } } PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !"); return boost::blank(); diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index 2b788a76cafe198abb9aed8ba842e37cc6ff73a6..48b54b5422de8c45e15a1b7040b78373dce8fa3a 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -27,13 +27,16 @@ limitations under the License. */ namespace paddle { namespace framework { -typedef boost::variant, - std::vector, std::vector, - std::vector>> +typedef boost::variant, std::vector, std::vector, + std::vector, + std::vector>, BlockDesc*> Attribute; typedef std::unordered_map AttributeMap; +ProgramDesc& GetProgramDesc(); + template AttrType AttrTypeID(); diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index c5d46622156c56acb98fb77e7db5ee7bca8c937a..0ec18de5b8a0e7cebdb91c30d2b45596b02dfa51 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -166,9 +166,8 @@ static std::unique_ptr BackwardRecursive( // If part of input gradient of that operator is not calculated, fill // zero variables to that input gradient. - net->AppendOp(OpRegistry::CreateOp("fill_zeros_like", - {{"Src", {prefix}}}, - {{"Dst", {grad_input}}}, {})); + net->AppendOp(OpRegistry::CreateOp("fill_zeros_like", {{"X", {prefix}}}, + {{"Y", {grad_input}}}, {})); } return false; }); diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index ad8003420dc14538d0dae9a1cb19d6459b154576..6932f5b989a3e21ebc44ec4fec9f5223f2547d7a 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -127,8 +127,8 @@ class FillZeroOpMaker : public OpProtoAndCheckerMaker { public: FillZeroOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Src", "x"); - AddOutput("Dst", "out"); + AddInput("X", "x"); + AddOutput("Y", "out"); AddComment(""); } }; @@ -325,10 +325,10 @@ TEST(Backward, op_part_of_output_are_not_need) { auto &fill_zero = *net->ops_[0]; ASSERT_EQ("fill_zeros_like", fill_zero.Type()); - ASSERT_EQ(1UL, fill_zero.Inputs("Src").size()); - ASSERT_EQ("Z", fill_zero.Input("Src")); - ASSERT_EQ(1UL, fill_zero.Outputs("Dst").size()); - ASSERT_EQ(std::string("Z") + f::kZeroVarSuffix, fill_zero.Output("Dst")); + ASSERT_EQ(1UL, fill_zero.Inputs("X").size()); + ASSERT_EQ("Z", fill_zero.Input("X")); + ASSERT_EQ(1UL, fill_zero.Outputs("Y").size()); + ASSERT_EQ(std::string("Z") + f::kZeroVarSuffix, fill_zero.Output("Y")); auto &d_many_out = *net->ops_[1]; ASSERT_EQ("many_output_op_grad", d_many_out.Type()); diff --git a/paddle/framework/ddim.cc b/paddle/framework/ddim.cc index fc3d508553c0e966978b28d58127bdbff10d45f1..a3357867530c110df16a5f3ec8c799735206cc71 100644 --- a/paddle/framework/ddim.cc +++ b/paddle/framework/ddim.cc @@ -292,5 +292,13 @@ DDim flatten_to_2d(const DDim& src, int num_col_dims) { DDim flatten_to_1d(const DDim& src) { return make_ddim({product(src)}); } +DDim stride(const DDim& ddim) { + std::vector strides(ddim.size()); + strides[ddim.size() - 1] = 1; + for (int i = ddim.size() - 2; i >= 0; --i) { + strides[i] = strides[i + 1] * ddim[i + 1]; + } + return framework::make_ddim(strides); +} } // namespace framework } // namespace paddle diff --git a/paddle/framework/ddim.h b/paddle/framework/ddim.h index ca29e7e8c7776de6adf3e3b0e8f11f0d4d8487c3..4a871bb0a91ed4050847509cc3f24218bcd57142 100644 --- a/paddle/framework/ddim.h +++ b/paddle/framework/ddim.h @@ -121,6 +121,7 @@ DDim flatten_to_2d(const DDim& src, int num_col_dims); DDim flatten_to_1d(const DDim& src); +DDim stride(const DDim& ddim); } // namespace framework } // namespace paddle diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index dfcb5fb6210a08f35193b83e3b5f7cee92f618d7..cf83d4cec312ac16366d84f897e7dc4784596ae8 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -23,6 +23,9 @@ enum AttrType { FLOATS = 4; STRINGS = 5; INT_PAIRS = 6; + BOOLEAN = 7; + BOOLEANS = 8; + BLOCK = 9; } message IntPair { @@ -44,6 +47,9 @@ message OpDesc { repeated float floats = 7; repeated string strings = 8; repeated IntPair int_pairs = 9; + optional bool b = 10; + repeated bool bools = 11; + optional int32 block_idx = 12; }; message Var { @@ -108,3 +114,12 @@ message VarDesc { required string name = 1; optional LoDTensorDesc lod_tensor = 2; } + +message BlockDesc { + required int32 idx = 1; + required int32 parent_idx = 2; + repeated VarDesc vars = 3; + repeated OpDesc ops = 4; +} + +message ProgramDesc { repeated BlockDesc blocks = 1; } diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 39b27d9b9583fe520a5d5d1254a1ea04e5f59ce2..07bbdf9416c432052b3222757a61ac4bfd70fe14 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -4,7 +4,7 @@ PaddlePaddle's RNN doesn't require that all instances have the same length. To ## Challenge of Variable-length Inputs -People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 32x32xO-dimensional tensor T and the 10x32x32 Tensor. +People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 10xOx32-dimensional tensor T and the 10x32x32 Tensor. Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths. @@ -54,7 +54,7 @@ In summary, as long as that the essential elements (words or images) have the s - The first dimension size L has an additonal property -- a LoD index as a nested vector: ```c++ - typedef std::vector > LoD; + typedef std::vector> LoD; ``` - The LoD index is not necessary when there are only two levels and all elements of the second level have length 1. @@ -100,7 +100,7 @@ Let's go on slicing this slice. Its <1,1>-slice is The algorithm, with over-simplified data structure, is defined as ```c++ -typedef vector > LoD; +typedef std::vector> LoD; struct LoDTensor { LoD lod_; diff --git a/paddle/framework/op_proto_maker.cc b/paddle/framework/op_proto_maker.cc new file mode 100644 index 0000000000000000000000000000000000000000..151d61d5b175535509306d028027c7bc19abce81 --- /dev/null +++ b/paddle/framework/op_proto_maker.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +namespace paddle { +namespace framework { + +void OpProtoAndCheckerMaker::Validate() { + validated_ = true; + CheckNoDuplicatedInOutAttrs(); +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( + const std::string& name, const std::string& comment) { + auto* input = proto_->add_inputs(); + input->set_name(name); + input->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{input}; +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( + const std::string& name, const std::string& comment) { + auto* output = proto_->add_outputs(); + output->set_name(name); + output->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{output}; +} + +void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { + std::unordered_set names; + auto checker = [&](const std::string& name) { + PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); + names.insert(name); + }; + for (auto& attr : proto_->attrs()) { + checker(attr.name()); + } + for (auto& input : proto_->inputs()) { + checker(input.name()); + } + for (auto& output : proto_->outputs()) { + checker(output.name()); + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker.h b/paddle/framework/op_proto_maker.h new file mode 100644 index 0000000000000000000000000000000000000000..4d55a37db9f0a3deac7b3489c8bc288ea41f4799 --- /dev/null +++ b/paddle/framework/op_proto_maker.h @@ -0,0 +1,88 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/attribute.h" +#include "paddle/framework/framework.pb.h" + +namespace paddle { +namespace framework { + +// this class not only make proto but also init attribute checkers. +class OpProtoAndCheckerMaker { + public: + OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : proto_(proto), op_checker_(op_checker) {} + + virtual ~OpProtoAndCheckerMaker() { + PADDLE_ENFORCE(validated_, "should call Validate after build"); + } + + void Validate(); + + protected: + struct VariableBuilder { + OpProto::Var* var_; + + VariableBuilder& AsDuplicable() { + var_->set_duplicable(true); + return *this; + } + + VariableBuilder& AsIntermediate() { + var_->set_intermediate(true); + return *this; + } + + VariableBuilder& NotInGradient() { + var_->set_not_in_gradient(true); + return *this; + } + }; + + VariableBuilder AddInput(const std::string& name, const std::string& comment); + + VariableBuilder AddOutput(const std::string& name, + const std::string& comment); + + template + TypedAttrChecker& AddAttr(const std::string& name, + const std::string& comment, + bool generated = false) { + auto* attr = proto_->add_attrs(); + attr->set_name(name); + attr->set_comment(comment); + attr->set_generated(generated); + attr->set_type(AttrTypeID()); + return op_checker_->AddAttrChecker(name); + } + + void AddComment(const std::string& comment) { proto_->set_comment(comment); } + + private: + void CheckNoDuplicatedInOutAttrs(); + + OpProto* proto_; + OpAttrChecker* op_checker_; + bool validated_{false}; +}; + +class NOPMaker : public OpProtoAndCheckerMaker { + public: + NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) {} +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker_test.cc b/paddle/framework/op_proto_maker_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..b01e30f75371ca4aa63dae86ddfb966b1d4c7830 --- /dev/null +++ b/paddle/framework/op_proto_maker_test.cc @@ -0,0 +1,51 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +#include "gtest/gtest.h" + +class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestAttrProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddAttr("scale", "scale of test op"); + AddAttr("scale", "scale of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedAttr) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} + +class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestInOutProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of test op"); + AddInput("input", "input of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedInOut) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} \ No newline at end of file diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 572dff860a306bb03ba9e6702fec85e4a2ea1b54..90077d0192421f3678a049a723972fcb1e8d67af 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -24,6 +24,7 @@ limitations under the License. */ #include "paddle/framework/framework.pb.h" #include "paddle/framework/grad_op_builder.h" #include "paddle/framework/op_info.h" +#include "paddle/framework/op_proto_maker.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index f8a64a786611ef872dbbfced10919e00c4d46715..a75fd49a470fa5f9ce7a47ecd03c1084f9b48c27 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -207,64 +207,25 @@ const std::vector InferShapeContext::MultiInput( } template <> -Tensor* ExecutionContext::Output(const std::string& name) const { - auto* var = OutputVar(name); - return var == nullptr ? nullptr : const_cast(GetTensorFromVar(var)); +Tensor* InferShapeContext::Output(const std::string& name) const { + auto var = OutputVar(name); + return var == nullptr ? nullptr : var->GetMutable(); } template <> -std::vector ExecutionContext::MultiOutput( +std::vector InferShapeContext::MultiOutput( const std::string& name) const { auto names = op().Outputs(name); std::vector res; res.reserve(names.size()); std::transform(names.begin(), names.end(), std::back_inserter(res), [&](const std::string& sub_name) { - auto var = scope().FindVar(sub_name); - return var == nullptr - ? nullptr - : const_cast(GetTensorFromVar(var)); + auto var = scope_.FindVar(sub_name); + return var == nullptr ? nullptr + : var->GetMutable(); }); return res; } -void OpProtoAndCheckerMaker::Validate() { - validated_ = true; - CheckNoDuplicatedInOutAttrs(); -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( - const std::string& name, const std::string& comment) { - auto* input = proto_->add_inputs(); - input->set_name(name); - input->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{input}; -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( - const std::string& name, const std::string& comment) { - auto* output = proto_->add_outputs(); - output->set_name(name); - output->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{output}; -} - -void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { - std::unordered_set names; - auto checker = [&](const std::string& name) { - PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); - names.insert(name); - }; - for (auto& attr : proto_->attrs()) { - checker(attr.name()); - } - for (auto& input : proto_->inputs()) { - checker(input.name()); - } - for (auto& output : proto_->outputs()) { - checker(output.name()); - } -} - } // namespace framework } // namespace paddle diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index b7c9c39402d57daf0aec97d98535ac8a8d9c0150..2d6d5510ef6dc83f1a016be6ff123f0b9bcaf230 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -167,71 +167,6 @@ class NOP : public OperatorBase { } }; -// this class not only make proto but also init attribute checkers. -class OpProtoAndCheckerMaker { - public: - OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) - : proto_(proto), op_checker_(op_checker) {} - - ~OpProtoAndCheckerMaker() { - PADDLE_ENFORCE(validated_, "should call Validate after build"); - } - - void Validate(); - - protected: - struct VariableBuilder { - OpProto::Var* var_; - - VariableBuilder& AsDuplicable() { - var_->set_duplicable(true); - return *this; - } - - VariableBuilder& AsIntermediate() { - var_->set_intermediate(true); - return *this; - } - - VariableBuilder& NotInGradient() { - var_->set_not_in_gradient(true); - return *this; - } - }; - - VariableBuilder AddInput(const std::string& name, const std::string& comment); - - VariableBuilder AddOutput(const std::string& name, - const std::string& comment); - - template - TypedAttrChecker& AddAttr(const std::string& name, - const std::string& comment, - bool generated = false) { - auto* attr = proto_->add_attrs(); - attr->set_name(name); - attr->set_comment(comment); - attr->set_generated(generated); - attr->set_type(AttrTypeID()); - return op_checker_->AddAttrChecker(name); - } - - void AddComment(const std::string& comment) { proto_->set_comment(comment); } - - private: - void CheckNoDuplicatedInOutAttrs(); - - OpProto* proto_; - OpAttrChecker* op_checker_; - bool validated_{false}; -}; - -class NOPMaker : public OpProtoAndCheckerMaker { - public: - NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) {} -}; - class InferShapeContext { public: InferShapeContext(const OperatorBase& op, const Scope& scope) @@ -277,9 +212,9 @@ class InferShapeContext { return res; } - std::vector MultiOutputVar(const std::string& name) const { + std::vector MultiOutputVar(const std::string& name) const { auto names = op_.Outputs(name); - std::vector res; + std::vector res; res.reserve(names.size()); std::transform(names.begin(), names.end(), std::back_inserter(res), [this](const std::string& name) { @@ -336,6 +271,20 @@ class InferShapeContext { return &var->Get(); } + void ShareLoD(const std::string& in, const std::string& out, size_t i = 0, + size_t j = 0) const { + PADDLE_ENFORCE_LT(i, InputSize(in)); + PADDLE_ENFORCE_LT(j, OutputSize(out)); + auto* in_var = MultiInputVar(in)[i]; + auto* out_var = MultiOutputVar(out)[j]; + if (!in_var->IsType()) return; + PADDLE_ENFORCE(out_var->IsType(), + "The %d-th output of Output(%s) must be LoDTensor.", j, out); + auto in_tensor = in_var->Get(); + auto* out_tensor = out_var->GetMutable(); + out_tensor->set_lod(in_tensor.lod()); + } + private: const OperatorBase& op_; const Scope& scope_; @@ -348,6 +297,13 @@ template <> const std::vector InferShapeContext::MultiInput( const std::string& name) const; +template <> +Tensor* InferShapeContext::Output(const std::string& name) const; + +template <> +std::vector InferShapeContext::MultiOutput( + const std::string& name) const; + template struct EigenDeviceConverter; @@ -380,38 +336,10 @@ class ExecutionContext : public InferShapeContext { return device_context_; } - // redefine Output function, - // use Variable::Get instead of Variable::GetMutable - template - T* Output(const std::string& name) const { - auto var = OutputVar(name); - return var == nullptr ? nullptr : const_cast(&var->Get()); - } - - // redefine MultiOutput function. - // use Variable::Get instead of Variable::GetMutable - template - std::vector MultiOutput(const std::string& name) const { - auto names = op().Outputs(name); - std::vector res; - res.reserve(names.size()); - std::transform( - names.begin(), names.end(), std::back_inserter(res), - [&](const std::string& sub_name) { return Output(sub_name); }); - return res; - } - private: const platform::DeviceContext& device_context_; }; -template <> -Tensor* ExecutionContext::Output(const std::string& name) const; - -template <> -std::vector ExecutionContext::MultiOutput( - const std::string& name) const; - class OpKernel { public: /** diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 20bbb11896a4c6f11079669f0b25773f6460594d..0beab0fac5b94c78121261d2661a6f969289afc4 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -264,37 +264,3 @@ TEST(Operator, Clone) { auto b = a.Clone(); ASSERT_EQ(a.Type(), b->Type()); } - -class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestAttrProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddAttr("scale", "scale of test op"); - AddAttr("scale", "scale of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedAttr) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} - -class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestInOutProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("input", "input of test op"); - AddInput("input", "input of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedInOut) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} \ No newline at end of file diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index 2ba3f8ed355b48800cfa4180e4e8a94f2c9958a9..c93b03e48130afe9568089b6a7586c4185d1d5b4 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -58,6 +58,8 @@ class Scope { /// nullptr if cannot find. Variable* FindVar(const std::string& name) const; + const Scope& parent() const { return *parent_; } + /// Find the scope or an ancestor scope that contains the given variable. const Scope* FindScope(const Variable* var) const; diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index ed166935f76be9d25062b5e69536c7b7ac19045d..6d2c14f4c47afb755b1c74f6dc4dd10ab25ed191 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -130,15 +130,19 @@ inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const { PADDLE_ENFORCE_LE(end_idx, dims_[0], "Slice end index is out of bound."); PADDLE_ENFORCE_LT(begin_idx, end_idx, "Begin index must be less than end index."); - PADDLE_ENFORCE_NE(dims_[0], 1, "Can not slice a tensor with dims_[0] = 1."); - size_t base = numel() / dims_[0]; - Tensor dst; - dst.holder_ = holder_; - DDim dst_dims = dims_; - dst_dims[0] = end_idx - begin_idx; - dst.Resize(dst_dims); - dst.offset_ = offset_ + begin_idx * base * sizeof(T); - return dst; + + if (dims_[0] == 1) { + return *this; + } else { + size_t base = numel() / dims_[0]; + Tensor dst; + dst.holder_ = holder_; + DDim dst_dims = dims_; + dst_dims[0] = end_idx - begin_idx; + dst.Resize(dst_dims); + dst.offset_ = offset_ + begin_idx * base * sizeof(T); + return dst; + } } inline Tensor& Tensor::Resize(const DDim& dims) { diff --git a/paddle/gserver/activations/MKLDNNActivation.h b/paddle/gserver/activations/MKLDNNActivation.h index bda9bbebe5600dbe26d11ff32058f7b2647b763e..86ffe387366409d81a91740cc8cea886e618f7e2 100644 --- a/paddle/gserver/activations/MKLDNNActivation.h +++ b/paddle/gserver/activations/MKLDNNActivation.h @@ -131,8 +131,9 @@ public: fwdPD_.reset(new eltwise_fwd::primitive_desc(fwdDesc, eng)); // use inplace for forward but save input value before submit inVal_ = val_; - if (act.grad) { - // only copy when need do backward + copyInVal_ = nullptr; + if (act.grad && algo == mkldnn::algorithm::eltwise_tanh) { + // tanh need save src input for backward inVal_ = MKLDNNMatrix::create(nullptr, val_->getPrimitiveDesc()); copyInVal_ = std::make_shared(*val_, *inVal_); CHECK(copyInVal_) << "should not be emptry"; diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index 2647cb600653b4f43322016afb231a55f4db5642..88b047c89bd40aba1afc456c22a2870c62989c1c 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -449,13 +449,14 @@ void MKLDNNConvLayer::resetOutGrad( cvtOutGrad_ = nullptr; if (!outputIsOnlyMKLDNN()) { const MatrixPtr& cpuOut = getOutput(CPU_DEVICE).grad; + outMat->setData(cpuOut->getData()); // same PrimitiveDesc with cpuInVal_ CHECK(cpuOutVal_); cpuOutGrad_ = MKLDNNMatrix::create(cpuOut, cpuOutVal_->getPrimitiveDesc()); if (cpuOutGrad_->getPrimitiveDesc() == out->getPrimitiveDesc()) { - outMat->setData(cpuOut->getData()); out = cpuOutGrad_; } else { + out = MKLDNNMatrix::create(nullptr, wgtPD->diff_dst_primitive_desc()); cvtOutGrad_ = MKLDNNMatrix::createReorder(cpuOutGrad_, out); CHECK(cvtOutGrad_); } diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index 66b358bcea53f61ddcc15323704fa9f154fb2a73..afd092666bf8b8a3389b36aa1f0edb256a9968e6 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -232,6 +232,7 @@ void MKLDNNFcLayer::resetBwdBuffers(MKLDNNMatrixPtr& in, void MKLDNNFcLayer::resetOutGrad(MKLDNNMatrixPtr& out) { // TODO(TJ): merge outgrad int device = outputIsOnlyMKLDNN() ? MKLDNN_DEVICE : CPU_DEVICE; + output_.grad->setData(getOutput(device).grad->getData()); // for MKLDNN device: // can not directly cast outputgrad to mkldnnmatrix, // since each layer can not write the inputgrad to mkldnn inputgrad. diff --git a/paddle/gserver/layers/MKLDNNLayer.h b/paddle/gserver/layers/MKLDNNLayer.h index c4e4a6874e6fdb491c344c70dfea422dc0924cd9..d8555a833187ddf64b096135e920e5be2b3a8c2f 100644 --- a/paddle/gserver/layers/MKLDNNLayer.h +++ b/paddle/gserver/layers/MKLDNNLayer.h @@ -141,18 +141,16 @@ public: } void backward(const UpdateCallback& callback) override { - /* Do derivation */ { + if (needResetBwd_) { + resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); + needResetBwd_ = false; + } + { REGISTER_TIMER_INFO("BpActTimer", getName().c_str()); backwardActivation(); } - { REGISTER_TIMER_INFO("mkldnn_bwdTimer", getName().c_str()); - if (needResetBwd_) { - resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); - needResetBwd_ = false; - } - stream_->submit(pipelineBwd_); } diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 406181370faf90d29167b62173ce4c8af44d243e..1bfbbde4246a10eaf86693a6a2f237f390966db3 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -26,17 +26,26 @@ DECLARE_bool(thread_local_rand_use_global_seed); DECLARE_bool(use_gpu); DECLARE_bool(use_mkldnn); -struct testFCDesc { +#define RUN_MKLDNN_TEST(DNN_CONFIG, REF_CONFIG, DESC) \ + MKLDNNTester tester; \ + for (auto bs : {DESC.bs, 1}) { \ + tester.run(DNN_CONFIG, REF_CONFIG, bs, DESC.ih, DESC.iw); \ + } + +#define RUN_MKLDNN_TEST_LAYER(DNN_CONFIG, REF_TYPE, DESC) \ + TestConfig ref = DNN_CONFIG; \ + ref.layerConfig.set_type(REF_TYPE); \ + RUN_MKLDNN_TEST(DNN_CONFIG, ref, DESC) + +struct testFcDesc { int bs; int ic; int oc; int ih, iw; // oh == ow == 1 }; -void testFcLayer(const testFCDesc& pm) { - const std::string compareTypes[] = {"mkldnn_fc", "fc"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNFcConfig(TestConfig& cfg, const testFcDesc& pm) { + cfg.layerConfig.set_type("mkldnn_fc"); cfg.layerConfig.set_size(pm.oc); cfg.inputDefs.push_back( {INPUT_DATA, @@ -44,25 +53,25 @@ void testFcLayer(const testFCDesc& pm) { /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), /* size of weight= */ size_t(pm.oc * pm.ic * pm.ih * pm.iw)}); cfg.layerConfig.add_inputs(); +} - MKLDNNTester tester; +void testFcLayer(const testFcDesc& pm) { + TestConfig dnnConfig; + getMKLDNNFcConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "fc", pm) } } TEST(MKLDNNLayer, FcLayer) { - testFcLayer({/*bs*/ 2, /*ic*/ 2, /*oc*/ 3, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 3, /*ic*/ 7, /*oc*/ 19, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 8, /*ic*/ 16, /*oc*/ 32, /*ih*/ 13, /*iw*/ 13}); - testFcLayer({/*bs*/ 4, /*ic*/ 12, /*oc*/ 18, /*ih*/ 13, /*iw*/ 11}); - testFcLayer({/*bs*/ 2, /*ic*/ 64, /*oc*/ 32, /*ih*/ 16, /*iw*/ 16}); - testFcLayer({/*bs*/ 15, /*ic*/ 3, /*oc*/ 6, /*ih*/ 16, /*iw*/ 16}); + /* bs, ic, ih, iw, oc */ + testFcLayer({2, 2, 1, 1, 3}); + testFcLayer({3, 7, 1, 1, 19}); + testFcLayer({8, 16, 13, 13, 32}); + testFcLayer({4, 12, 13, 13, 18}); + testFcLayer({2, 64, 16, 16, 32}); + testFcLayer({15, 3, 16, 16, 6}); } struct testConvDesc { @@ -75,13 +84,10 @@ struct testConvDesc { int dh, dw; }; -void testConvLayer(const testConvDesc& pm) { - const std::string compareTypes[] = {"mkldnn_conv", "exconv"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNConvConfig(TestConfig& cfg, const testConvDesc& pm) { + cfg.layerConfig.set_type("mkldnn_conv"); cfg.layerConfig.set_num_filters(pm.oc); cfg.layerConfig.set_size(pm.oc * pm.oh * pm.ow); - // cfg.layerConfig.set_partial_sum(1); // TODO: check it cfg.layerConfig.set_shared_biases(true); cfg.inputDefs.push_back( {INPUT_DATA, @@ -115,15 +121,14 @@ void testConvLayer(const testConvDesc& pm) { int oh = outputSize(pm.ih, fh, pm.ph, pm.sh, true); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testConvLayer(const testConvDesc& pm) { + TestConfig dnnConfig; + getMKLDNNConvConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "exconv", pm) } } @@ -143,7 +148,7 @@ TEST(MKLDNNLayer, ConvLayer) { } struct testPoolDesc { - int bs, ch; // input channel and output channel are the same + int bs, ic; // input channel and output channel are the same int ih, iw; int oh, ow; int fh, fw; @@ -151,19 +156,18 @@ struct testPoolDesc { int sh, sw; }; -void testPoolLayer(const testPoolDesc& pm) { - const std::string compareTypes[] = {"mkldnn_pool", "pool"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); - cfg.layerConfig.set_size(pm.ch * pm.oh * pm.ow); +static void getMKLDNNPoolConfig(TestConfig& cfg, const testPoolDesc& pm) { + cfg.layerConfig.set_type("mkldnn_pool"); + cfg.layerConfig.set_size(pm.ic * pm.oh * pm.ow); cfg.inputDefs.push_back( {INPUT_DATA, "layer_0", - /* size of input layer= */ size_t(pm.ch * pm.ih * pm.iw), + /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), 0}); LayerInputConfig* input = cfg.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - pool->set_channels(pm.ch); + pool->set_pool_type("avg-projection"); + pool->set_channels(pm.ic); pool->set_img_size(pm.iw); pool->set_img_size_y(pm.ih); pool->set_output_x(pm.ow); @@ -179,20 +183,21 @@ void testPoolLayer(const testPoolDesc& pm) { int ow = outputSize(pm.iw, pm.fw, pm.pw, pm.sw, false); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testPoolLayer(const testPoolDesc& pm) { + TestConfig dnnConfig; + getMKLDNNPoolConfig(dnnConfig, pm); + LayerInputConfig* input = dnnConfig.layerConfig.mutable_inputs(0); + PoolConfig* pool = input->mutable_pool_conf(); for (auto type : {"max-projection", "avg-projection"}) { pool->set_pool_type(type); - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + RUN_MKLDNN_TEST_LAYER(dnnConfig, "pool", pm) } } TEST(MKLDNNLayer, PoolLayer) { - /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw*/ + /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw */ testPoolLayer({2, 1, 4, 4, 2, 2, 3, 3, 0, 0, 2, 2}); testPoolLayer({10, 8, 16, 16, 8, 8, 2, 2, 0, 0, 2, 2}); testPoolLayer({4, 2, 5, 5, 3, 3, 3, 3, 1, 1, 2, 2}); @@ -204,44 +209,36 @@ TEST(MKLDNNLayer, PoolLayer) { } struct testActDesc { - int bs, ch; - int ih, iw; + int bs, ic, ih, iw; }; static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { cfg.biasSize = 0; cfg.layerConfig.set_type("addto"); - cfg.layerConfig.set_size(pm.ch * pm.ih * pm.iw); - cfg.inputDefs.push_back( - {INPUT_DATA, - "layer_0", - /* size of input layer= */ size_t(pm.ch * pm.ih * pm.iw), - 0}); + size_t layerSize = pm.ih * pm.ih * pm.iw; + cfg.layerConfig.set_size(layerSize); + cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); cfg.layerConfig.add_inputs(); } -void testActivation(std::string& type, const testActDesc& pm) { - const std::string compareTypes[] = {type, type.erase(0, 7)}; +void testActivation(std::string& actType, const testActDesc& pm) { + // TODO(TJ): mkldnn_softmax not implemented, paddle do not have elu activation + if (actType == "mkldnn_softmax" || actType == "mkldnn_elu") { + return; + } + const std::string compareTypes[] = {actType, actType.erase(0, 7)}; TestConfig cfg; getAddtoConfig(cfg, pm); - TestConfig ref = cfg; cfg.layerConfig.set_active_type(compareTypes[0]); ref.layerConfig.set_active_type(compareTypes[1]); - MKLDNNTester tester; - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + RUN_MKLDNN_TEST(cfg, ref, pm) } TEST(MKLDNNActivation, Activations) { auto types = MKLDNNActivation::getAllRegisteredTypes(); - // TODO(TJ): mkldnn_softmax not implemented, paddle do not have elu activation - std::set excluded{"mkldnn_softmax", "mkldnn_elu"}; for (auto type : types) { - if (excluded.count(type)) { - continue; - } + /* bs, c, h, w*/ testActivation(type, {16, 64, 32, 32}); } } diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index e3e934bcccd1a5f34d88a2f33f3708a46ddabe05..f8b0bce6815ff17a60ef64b0eec34a7cc9d16e72 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -55,6 +55,13 @@ function(op_library TARGET) set(pybind_flag 1) endif() + # activation_op contains several operators + if ("${TARGET}" STREQUAL "activation_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(sigmoid);\n") + endif() + # pybind USE_NO_KERNEL_OP file(READ ${TARGET}.cc TARGET_CONTENT) string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}") @@ -96,3 +103,4 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) +cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc index 0c813748b2989a8f0c00a359345747242dd21dd8..70e4f9da1221ab300e2b507a3da2f7c5da93f2e4 100644 --- a/paddle/operators/accuracy_op.cc +++ b/paddle/operators/accuracy_op.cc @@ -39,7 +39,8 @@ class AccuracyOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(inference->dims()[0], label->dims()[0], "inference size must be the same as label size"); - ctx.Output("Accuracy")->Resize({1}); + ctx.Output("Accuracy")->Resize({1}); + ctx.ShareLoD("Inference", /*->*/ "Accuracy"); } }; @@ -54,11 +55,15 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker { // TODO(typhoonzero): AddInput("Weight", ... AddOutput("Accuracy", "The accuracy of current batch"); - AddComment( - R"DOC(Accuracy. It will print accuracy rate for classification. + AddComment(R"DOC( +Accuracy. It will print accuracy rate for classification. The accuracy is: .. math:: -accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples})DOC"); +accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples}) + +Both the input `Inference` and `Label` can carry the LoD (Level of Details) +information, or not. But the output only shares the LoD with input `Inference`. +)DOC"); } }; diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..06654702bc42cc7cf4917b00693334b1d36ce371 --- /dev/null +++ b/paddle/operators/activation_op.cc @@ -0,0 +1,307 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/activation_op.h" + +namespace paddle { +namespace operators { + +class ActivationOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output("Y")->Resize( + ctx.Input("X")->dims()); + ctx.ShareLoD("X", /*->*/ "Y"); + } +}; + +class ActivationOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output(framework::GradVarName("X")) + ->Resize(ctx.Input("Y")->dims()); + } +}; + +class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SigmoidOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sigmoid operator"); + AddOutput("Y", "Output of Sigmoid operator"); + AddComment("Sigmoid activation operator, sigmoid = 1 / (1 + exp(-x))"); + } +}; + +class ExpOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ExpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Exp operator"); + AddOutput("Y", "Output of Exp operator"); + AddComment("Exp activation operator, exp(x) = e^x"); + } +}; + +class ReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Relu operator"); + AddOutput("Y", "Output of Relu operator"); + AddComment("Relu activation operator, relu(x) = max(x, 0)"); + } +}; + +class TanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Tanh operator"); + AddOutput("Y", "Output of Tanh operator"); + AddComment( + "Tanh activation operator, tanh = (exp(x) - exp(-x)) / (exp(x) + " + "exp(-x))"); + } +}; + +class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SqrtOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sqrt operator"); + AddOutput("Y", "Output of Sqrt operator"); + AddComment("Sqrt activation operator, sqrt(x) = x^(1/2)"); + } +}; + +class AbsOpMaker : public framework::OpProtoAndCheckerMaker { + public: + AbsOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Abs operator"); + AddOutput("Y", "Output of Abs operator"); + AddComment("Abs activation operator, abs(x) = |x|"); + } +}; + +class ReciprocalOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReciprocalOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Reciprocal operator"); + AddOutput("Y", "Output of Reciprocal operator"); + AddComment("Reciprocal activation operator, reciprocal(x) = 1 / x"); + } +}; + +class LogOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LogOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Log operator"); + AddOutput("Y", "Output of Log operator"); + AddComment("Log activation operator, log(x) = natural logarithm of x"); + } +}; + +class SquareOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SquareOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Square operator"); + AddOutput("Y", "Output of Square operator"); + AddComment("Square activation operator, square(x) = x^2"); + } +}; + +template +class BReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + BReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of BRelu operator"); + AddOutput("Y", "Output of BRelu operator"); + AddComment("BRelu activation operator, brelu = max(min(x, t_min), t_max)"); + AddAttr("t_min", "The min marginal value of BRelu") + .SetDefault(static_cast(0)); + AddAttr("t_max", "The max marginal value of BRelu") + .SetDefault(static_cast(24)); + } +}; + +template +class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SoftReluOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of SoftRelu operator"); + AddOutput("Y", "Output of SoftRelu operator"); + AddComment( + "SoftRelu activation operator, soft_relu = log(1 + exp(max(min(x, " + "threshold), threshold)))"); + AddAttr("threshold", "The threshold value of SoftRelu") + .SetDefault(static_cast(40)); + } +}; + +template +class PowOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PowOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Pow operator"); + AddOutput("Y", "Output of Pow operator"); + AddComment("Pow activation operator, pow(x, factor) = x^factor"); + AddAttr("factor", "The exponential factor of Pow") + .SetDefault(static_cast(1)); + } +}; + +template +class STanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + STanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of STanh operator"); + AddOutput("Y", "Output of STanh operator"); + AddComment("STanh activation operator, stanh = b * tanh(a * x)"); + AddAttr("scale_a", "The scale parameter of a for the input") + .SetDefault(static_cast(2 / 3)); + AddAttr("scale_b", "The scale parameter of b for the input") + .SetDefault(static_cast(1.7159)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(sigmoid, ops::ActivationOp, ops::SigmoidOpMaker, sigmoid_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP(exp, ops::ActivationOp, ops::ExpOpMaker, exp_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(exp_grad, + ops::ActivationGradKernel); + +REGISTER_OP(relu, ops::ActivationOp, ops::ReluOpMaker, relu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP(abs, ops::ActivationOp, ops::AbsOpMaker, abs_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP(reciprocal, ops::ActivationOp, ops::ReciprocalOpMaker, + reciprocal_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP(log, ops::ActivationOp, ops::LogOpMaker, log_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP(square, ops::ActivationOp, ops::SquareOpMaker, square_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_CPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, + soft_relu_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_CPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker, pow_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_CPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker, stanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_CPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..feed1302b292a546f88fa35457c86aa2cfdaa307 --- /dev/null +++ b/paddle/operators/activation_op.cu @@ -0,0 +1,100 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/activation_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(exp_grad, + ops::ActivationGradKernel); +REGISTER_OP_GPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP_GPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_GPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP_GPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_GPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP_GPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_GPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP_GPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_GPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h new file mode 100644 index 0000000000000000000000000000000000000000..15f8afb4ba45cc989fe7576b82b8bf853b1df7de --- /dev/null +++ b/paddle/operators/activation_op.h @@ -0,0 +1,353 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class ActivationKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y); + } +}; + +template +class ActivationGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y, dy, dx); + } +}; + +// sigmoid(x) = 1 / (1 + exp(-x)) +template +struct SigmoidFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / (static_cast(1) + (-x).exp()); + } +}; + +template +struct SigmoidGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y * (static_cast(1) - y); + } +}; + +// exp(x) = e^x +struct ExpFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.exp(); + } +}; + +struct ExpGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y; + } +}; + +// relu(x) = max(x, 0) +template +struct ReluFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.cwiseMax(static_cast(0)); + } +}; + +template +struct ReluGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (x > static_cast(0)).template cast(); + } +}; + +// tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) +struct TanhFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.tanh(); + } +}; + +template +struct TanhGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) - y * y); + } +}; + +// sqrt(x) = x^(1/2) +struct SqrtFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.sqrt(); + } +}; + +template +struct SqrtGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + const Y y_conj = Eigen::numext::conj(y); + dx.device(d) = static_cast(0.5) * dy / y_conj; + } +}; + +// abs(x) = |x| +struct AbsFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.abs(); + } +}; + +struct AbsGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * x.sign(); + } +}; + +// reciprocal(x) = 1 / x +template +struct ReciprocalFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / x; + } +}; + +template +struct ReciprocalGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(-1) * y * y; + } +}; + +// log(x) = natural logarithm of x +struct LogFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.log(); + } +}; + +template +struct LogGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) / x); + } +}; + +// square(x) = x^2 +struct SquareFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.square(); + } +}; + +template +struct SquareGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(2) * x; + } +}; + +template +class BReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.cwiseMax(t_min).cwiseMin(t_max); + } +}; + +template +class BReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * ((x > t_min) * (x < t_max)).template cast(); + } +}; + +template +class SoftReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto threshold = static_cast(context.Attr("threshold")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + auto temp = x.cwiseMax(-threshold).cwiseMin(threshold).eval(); + y.device(place) = (static_cast(1) + temp.exp()).log(); + } +}; + +template +class SoftReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto threshold = static_cast(context.Attr("threshold")); + dX->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dy = framework::EigenVector::Flatten(*dY); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + auto temp = ((x > -threshold) * (x < threshold)).template cast().eval(); + dx.device(place) = dy * (static_cast(1) - (-y).exp()) * temp; + } +}; + +template +class PowKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto factor = static_cast(context.Attr("factor")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.pow(factor); + } +}; + +template +class PowGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto factor = static_cast(context.Attr("factor")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * factor * x.pow(factor - static_cast(1)); + } +}; + +template +class STanhKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = scale_b * (scale_a * x).tanh(); + } +}; + +template +class STanhGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + auto temp = (scale_a * x).tanh() * (scale_a * x).tanh(); + dx.device(place) = dy * scale_a * scale_b * (static_cast(1) - temp); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc index e83c1efeaf897889d18a37a6bd2ca2f8f012db25..ed11d096974341022637676537793645f46738f0 100644 --- a/paddle/operators/add_op.cc +++ b/paddle/operators/add_op.cc @@ -33,7 +33,7 @@ class AddOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ctx.Input("X")->dims(), ctx.Input("Y")->dims(), "Two input of Add Op's dimension must be same."); - ctx.Output("Out")->Resize( + ctx.Output("Out")->Resize( ctx.Input("X")->dims()); } }; diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..e5a54bc4b226fd24337050fdd84b2de9c49f7949 --- /dev/null +++ b/paddle/operators/clip_op.cc @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_op.h" + +namespace paddle { +namespace operators { + +class ClipOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of ClipOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) of ClipOp should not be null."); + auto x_dims = ctx.Input("X")->dims(); + auto max = Attr("max"); + auto min = Attr("min"); + PADDLE_ENFORCE_LT(min, max, "max should be greater than min."); + ctx.Output("Out")->Resize(x_dims); + ctx.ShareLoD("X", /*->*/ "Out"); + } +}; + +template +class ClipOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ClipOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor)The input of clip op." + "The input should be a k-D tensor(k > 0 and k < 7)"); + AddOutput("Out", "(Tensor)The output of clip op with shape as input(X)"); + AddAttr( + "min", "(float)Minimum value, under which element is replaced by min."); + AddAttr( + "max", "(float)Maximum value, above which element is replaced by max"); + AddComment(R"DOC( +Clip operator limits the given input within an interval. The interval is +specified with arguments 'min' and 'max'. +)DOC"); + } +}; + +class ClipOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + if (x_grad != nullptr) { + x_grad->Resize(x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(clip, ops::ClipOp, ops::ClipOpMaker, clip_grad, + ops::ClipOpGrad); +REGISTER_OP_CPU_KERNEL(clip, + ops::ClipKernel); +REGISTER_OP_CPU_KERNEL(clip_grad, + ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.cu b/paddle/operators/clip_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..ca9701298fdae3fabe234925edaf9e4d775cc66e --- /dev/null +++ b/paddle/operators/clip_op.cu @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(clip, + ops::ClipKernel); +REGISTER_OP_GPU_KERNEL(clip_grad, + ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.h b/paddle/operators/clip_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ce1d4e1f460414e6e4acee4fa3207f309c55d86b --- /dev/null +++ b/paddle/operators/clip_op.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; +using platform::Transform; + +template +class ClipFunctor { + public: + explicit ClipFunctor(const T min, const T max) : min_(min), max_(max) {} + HOSTDEVICE T operator()(const T& x) const { + if (x < min_) + return min_; + else if (x > max_) + return max_; + else + return x; + } + + private: + T min_; + T max_; +}; + +template +class ClipGradFunctor { + public: + explicit ClipGradFunctor(const T min, const T max) : min_(min), max_(max) {} + HOSTDEVICE T operator()(const T& x, const T& y) const { + return (y > min_ && y < max_) ? x : 0; + } + + private: + T min_; + T max_; +}; + +template +class ClipKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max = context.Attr("max"); + auto min = context.Attr("min"); + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + T* out_data = out->mutable_data(context.GetPlace()); + const T* x_data = x->data(); + int64_t numel = x->numel(); + Transform trans; + trans(context.device_context(), x_data, x_data + numel, out_data, + ClipFunctor(min, max)); + } +}; + +template +class ClipGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max = context.Attr("max"); + auto min = context.Attr("min"); + auto* d_out = context.Input(framework::GradVarName("Out")); + auto* d_x = context.Output(framework::GradVarName("X")); + if (d_x != nullptr) { + auto* x = context.Input("X"); + int64_t numel = d_out->numel(); + auto* d_x_data = d_x->mutable_data(context.GetPlace()); + const T* d_out_data = d_out->data(); + const T* x_data = x->data(); + Transform trans; + trans(context.device_context(), d_out_data, d_out_data + numel, x_data, + d_x_data, ClipGradFunctor(min, max)); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc index 223bb0ffe6e75ce71919eb5f4cca06bedbb00764..07f847079e834716904dcc038d2097efd268bd3e 100644 --- a/paddle/operators/concat_op.cc +++ b/paddle/operators/concat_op.cc @@ -29,7 +29,7 @@ class ConcatOp : public framework::OperatorWithKernel { "Output(Out) of ConcatOp should not be null."); auto ins = ctx.MultiInput("X"); - auto *out = ctx.Output("Out"); + auto *out = ctx.Output("Out"); size_t axis = static_cast(ctx.Attr("axis")); size_t n = ins.size(); diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..c3281db0964de6d7dd6be629fbcc55cabb9fef9d --- /dev/null +++ b/paddle/operators/conv2d_op.cc @@ -0,0 +1,132 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/gemm_conv2d_op.h" + +namespace paddle { +namespace operators { + +int outputSize(int input_size, int filter_size, int padding, int stride) { + int output_size = (input_size - filter_size + 2 * padding) / stride + 1; + return output_size; +} + +class Conv2DOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Input"), + "Input(Input) of Conv2DOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Filter"), + "Input(Filter) of Conv2DOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Output"), + "Output(Output) of Conv2DOp should not be null."); + + auto in = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto out = ctx.Output("Output"); + std::vector strides = Attr>("strides"); + std::vector paddings = Attr>("paddings"); + int groups = Attr("groups"); + int input_channels = in->dims()[1]; + int output_channels = filter->dims()[0]; + + PADDLE_ENFORCE_EQ(in->dims().size(), 4, "Conv2DOp input should be 4-D."); + PADDLE_ENFORCE_EQ(filter->dims().size(), 4, + "Conv2DOp filter should be 4-D."); + PADDLE_ENFORCE_EQ(input_channels, filter->dims()[1] * groups, + "The number of input channels should be equal to filter " + "channels * groups."); + PADDLE_ENFORCE_EQ( + output_channels % groups, 0, + "The number of output channels should be divided by groups."); + + auto output_height = + outputSize(in->dims()[2], filter->dims()[2], paddings[0], strides[0]); + auto output_width = + outputSize(in->dims()[3], filter->dims()[3], paddings[1], strides[1]); + out->Resize( + {in->dims()[0], filter->dims()[0], output_height, output_width}); + } +}; + +class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Conv2DOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "The input tensor of convolution operator. " + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of channels, H and W is the height and width of image."); + AddInput( + "Filter", + "The filter tensor of convolution operator." + "The format of the filter tensor is MCHW, where M is the number of " + "output image channels, C is the number of input image channels, " + "H and W is height and width of filter. " + "If the groups attribute is greater than 1, C equal the number of " + "input image channels divided by the groups."); + AddOutput("Output", + "The output tensor of convolution operator." + "The format of output tensor is also NCHW."); + AddAttr>("strides", "strides of convolution operator.") + .SetDefault({1, 1}); + AddAttr>("paddings", "paddings of convolution operator.") + .SetDefault({0, 0}); + AddAttr( + "groups", + "group size of convolution operator. " + "Refer to grouped convolution in Alex Krizhevsky's paper: " + "when group=2, the first half of the filters are only connected to the " + "first half of the input channels, and the second half only connected " + "to the second half.") + .SetDefault(1); + AddComment(R"DOC( +The convolution operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. +)DOC"); + } +}; + +class Conv2DOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + auto in = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto d_in = ctx.Output(framework::GradVarName("Input")); + auto d_filter = + ctx.Output(framework::GradVarName("Filter")); + if (d_in) d_in->Resize(in->dims()); + if (d_filter) d_filter->Resize(filter->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(conv2d, ops::Conv2DOp, ops::Conv2DOpMaker, conv2d_grad, + ops::Conv2DOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv2d, ops::GemmConv2DKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_grad, ops::GemmConvGrad2DKernel); diff --git a/paddle/operators/conv2d_op.cu b/paddle/operators/conv2d_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..5df818ba0496a65502dde37fd1397ec56f8c1101 --- /dev/null +++ b/paddle/operators/conv2d_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/gemm_conv2d_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + conv2d, ops::GemmConv2DKernel); +REGISTER_OP_GPU_KERNEL( + conv2d_grad, ops::GemmConvGrad2DKernel); diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc index 72c446493684246959656dc048e7f0e761665423..b56ee2047b811e212b4bf74bf7fbba753a6bcb11 100644 --- a/paddle/operators/cos_sim_op.cc +++ b/paddle/operators/cos_sim_op.cc @@ -54,9 +54,10 @@ class CosSimOp : public framework::OperatorWithKernel { " just 1 (which will be broadcasted to match Input(X))."); // resize tensor - ctx.Output("Out")->Resize({x_dims[0], 1}); - ctx.Output("XNorm")->Resize({x_dims[0], 1}); - ctx.Output("YNorm")->Resize({y_dims[0], 1}); + ctx.Output("Out")->Resize({x_dims[0], 1}); + ctx.Output("XNorm")->Resize({x_dims[0], 1}); + ctx.Output("YNorm")->Resize({y_dims[0], 1}); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -81,10 +82,13 @@ Cosine Similarity Operator. The equation is: Out = X^T * Y / (sqrt(X^T * X) * sqrt(Y^T * Y)). -Input(X) and Input(Y) must have the same shape, except that the 1st dimension -of Input(Y) could be just 1 (different from Input(X)), which will be -broadcasted to match the shape of Input(X) before computing their cosine +The input `X` and `Y` must have the same shape, except that the 1st dimension +of input `Y` could be just 1 (different from input `X`), which will be +broadcasted to match the shape of input `X` before computing their cosine similarity. + +Both the input `X` and `Y` can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with input `X`. )DOC"); } }; @@ -139,10 +143,8 @@ class CosSimOpGrad : public framework::OperatorWithKernel { "Shape of Input(Out@Grad) must be [X.Dim(0), 1]."); // resize tensor - auto *x_grad = - ctx.Output(framework::GradVarName("X")); - auto *y_grad = - ctx.Output(framework::GradVarName("Y")); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + auto *y_grad = ctx.Output(framework::GradVarName("Y")); if (x_grad) x_grad->Resize(x_dims); if (y_grad) y_grad->Resize(y_dims); } diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..52a1123348b10e39bcfa1ba062c893e5f20ed862 --- /dev/null +++ b/paddle/operators/crop_op.cc @@ -0,0 +1,138 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/crop_op.h" +#include + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class CropOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of CropOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) of CropOp should not be null."); + auto x_dim = ctx.Input("X")->dims(); + auto *y = ctx.Input("Y"); + auto *out = ctx.Output("Out"); + if (y == nullptr) { + auto shape = Attr>("shape"); + PADDLE_ENFORCE_EQ( + int64_t(shape.size()), x_dim.size(), + "Shape size should be equal to dimention size of input tensor."); + std::vector tensor_shape(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + tensor_shape[i] = static_cast(shape[i]); + } + out->Resize(framework::make_ddim(tensor_shape)); + } else { + PADDLE_ENFORCE_EQ(framework::arity(x_dim), framework::arity(y->dims()), + "Tensor rank of both CropOp's " + "inputs must be same."); + out->Resize(y->dims()); + } + } +}; + +class CropOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CropOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input of pad op. " + "The input should be a k-D tensor(k > 0 and k < 7)"); + AddInput("Y", + "The input used as reference for cropping" + " with the same dimension as X. "); + AddOutput("Out", + "The output of crop op " + "with the same dimension as X."); + AddAttr>("offsets", + "A list describing offsets to be cropped." + "The size of offsets list should be as same as " + "dimension size of input X."); + AddAttr>("shape", + "A list describing the shape of output." + "The size of shape list should be as same as " + "dimension size of input X.") + .SetDefault(std::vector()); + AddComment(R"DOC( +Crop Operator. +Crop input into output, as specified by offsets and shape. + +There are two ways to set shape: +1. referenc input: crop input X as shape as reference input. + The dimension of reference input should + be as same as input X. +2. shape list: crop input X by shape described by a list. + The size of shape list should be as same as + dimension size of input X. + +The input should be a k-D tensor(k > 0 and k < 7). As an example: + +Given: + + X = [[0, 1, 2, 0, 0] + [0, 3, 4, 0, 0] + [0, 0, 0, 0, 0]] + +and + + offsets = [0, 1] + +and + + shape = [2, 2] + +then we get + + Out = [[1, 2], + [3, 4]] + +)DOC"); + } +}; + +class CropOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + if (x_grad != nullptr) { + x_grad->Resize(x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(crop, ops::CropOp, ops::CropOpMaker, crop_grad, ops::CropOpGrad); +REGISTER_OP_CPU_KERNEL(crop, ops::CropKernel); +REGISTER_OP_CPU_KERNEL(crop_grad, + ops::CropGradKernel); diff --git a/paddle/operators/crop_op.cu b/paddle/operators/crop_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..f8ee18a1d6e894cbb2d71dd4b6b459abeb076817 --- /dev/null +++ b/paddle/operators/crop_op.cu @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/crop_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(crop, ops::CropKernel); +REGISTER_OP_GPU_KERNEL(crop_grad, + ops::CropGradKernel); diff --git a/paddle/operators/crop_op.h b/paddle/operators/crop_op.h new file mode 100644 index 0000000000000000000000000000000000000000..2f40c059033ec649b29f6ecdee4fcedd128a63a6 --- /dev/null +++ b/paddle/operators/crop_op.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2016 CropdleCropdle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/strided_memcpy.h" + +namespace paddle { +namespace operators { // Internal + +template +using EigenTensor = framework::EigenTensor; +using framework::Tensor; + +template +class CropKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + const T* x_data = x->data(); + T* out_data = out->mutable_data(context.GetPlace()); + auto x_stride = framework::stride(x->dims()); + auto out_stride = framework::stride(out->dims()); + auto offsets = context.Attr>("offsets"); + PADDLE_ENFORCE_EQ( + x->dims().size(), offsets.size(), + "Offsets size should be equal to dimension size of input tensor."); + int64_t offset = 0; + for (int i = 0; i < offsets.size(); ++i) { + offset += (x_stride[i] * offsets[i]); + } + StridedMemcpy(context.device_context(), x_data + offset, x_stride, + out->dims(), out_stride, out_data); + } +}; + +template +void CropGradFunction(const framework::ExecutionContext& context) { + auto* d_x = context.Output(framework::GradVarName("X")); + if (d_x != nullptr) { + auto* d_out = context.Input(framework::GradVarName("Out")); + d_x->mutable_data(context.GetPlace()); + auto offsets = context.Attr>("offsets"); + Eigen::array, D> paddings; + for (int i = 0; i < D; ++i) { + paddings[i].first = offsets[i]; + paddings[i].second = d_x->dims()[i] - d_out->dims()[i] - offsets[i]; + } + auto d_x_tensor = EigenTensor::From(*d_x); + auto d_out_tensor = EigenTensor::From(*d_out); + d_x_tensor.device(context.GetEigenDevice()) = + d_out_tensor.pad(paddings, 0); + } +} + +template +class CropGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + size_t rank = + context.Input(framework::GradVarName("Out"))->dims().size(); + switch (rank) { + case 1: + CropGradFunction(context); + break; + case 2: + CropGradFunction(context); + break; + case 3: + CropGradFunction(context); + break; + case 4: + CropGradFunction(context); + break; + case 5: + CropGradFunction(context); + break; + case 6: + CropGradFunction(context); + break; + default: + PADDLE_THROW( + "CropOp only support tensors with no more than 6 dimensions."); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index 953367eb8bcd1282ab6c7e1189d778f0ce3da541..b11dc1472d153dd188a0b3553d6950774216a3fd 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -17,8 +17,6 @@ limitations under the License. */ namespace paddle { namespace operators { -using framework::LoDTensor; - class CrossEntropyOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -35,23 +33,21 @@ class CrossEntropyOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank must be 2."); PADDLE_ENFORCE_EQ(label->dims().size(), 2, "Input(Label)'s rank must be 2."); - // TODO(xinghai-sun): remove this check after swtiching to bool - PADDLE_ENFORCE(ctx.Attr("soft_label") == 0 || - ctx.Attr("soft_label") == 1); PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], "The 1st dimension of Input(X) and Input(Label) must " "be equal."); - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], - "If Attr(soft_label) == 1, The 2nd dimension of " + "If Attr(soft_label) == true, The 2nd dimension of " "Input(X) and Input(Label) must be equal."); } else { PADDLE_ENFORCE_EQ(label->dims()[1], 1, - "If Attr(soft_label) == 0, The 2nd dimension of " + "If Attr(soft_label) == false, The 2nd dimension of " "Input(Label) must be 1."); } - ctx.Output("Y")->Resize({x->dims()[0], 1}); + ctx.Output("Y")->Resize({x->dims()[0], 1}); + ctx.ShareLoD("X", /*->*/ "Y"); } }; @@ -74,9 +70,6 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(dy->dims().size(), 2, "Input(Y@Grad)'s rank must be 2."); PADDLE_ENFORCE_EQ(label->dims().size(), 2, "Input(Label)'s rank must be 2."); - // TODO(xinghai-sun): remove this check after swtiching to bool - PADDLE_ENFORCE(ctx.Attr("soft_label") == 0 || - ctx.Attr("soft_label") == 1); PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], "The 1st dimension of Input(X) and Input(Label) must " "be equal."); @@ -85,17 +78,17 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel { "be equal."); PADDLE_ENFORCE_EQ(dy->dims()[1], 1, "The 2nd dimension of Input(Y@Grad) must be 1."); - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], - "If Attr(soft_label) == 1, The 2nd dimension of " + "If Attr(soft_label) == true, The 2nd dimension of " "Input(X) and Input(Label) must be equal."); } else { PADDLE_ENFORCE_EQ(label->dims()[1], 1, - "If Attr(soft_label) == 0, The 2nd dimension of " + "If Attr(soft_label) == false, The 2nd dimension of " "Input(Label) must be 1."); } - auto dx = ctx.Output(framework::GradVarName("X")); + auto dx = ctx.Output(framework::GradVarName("X")); dx->Resize(x->dims()); } }; @@ -108,7 +101,8 @@ class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "The first input of CrossEntropyOp"); AddInput("Label", "The second input of CrossEntropyOp"); AddOutput("Y", "The output of CrossEntropyOp"); - AddAttr("soft_label", "Is soft label. Default zero.").SetDefault(0); + AddAttr("soft_label", "Is soft label. Default zero.") + .SetDefault(false); AddComment(R"DOC( CrossEntropy Operator. @@ -116,12 +110,12 @@ CrossEntropy Operator. It supports both standard cross-entropy and soft-label cross-entropy loss computation. 1) One-hot cross-entropy: - soft_label = 0, Label[i, 0] indicates the class index for sample i: + soft_label = False, Label[i, 0] indicates the class index for sample i: Y[i] = -log(X[i, Label[i]]) 2) Soft-label cross-entropy: - soft_label = 1, Label[i, j] indicates the soft label of class j + soft_label = True, Label[i, j] indicates the soft label of class j for sample i: Y[i] = \sum_j{-Label[i, j] * log(X[i, j])} @@ -133,6 +127,9 @@ computation. As a special case of 2), when each row of Input(Label) has only one non-zero element (equals 1), soft-label cross-entropy degenerates to a one-hot cross-entropy with one-hot label representation. + +Both the input `X` and `Label` can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with input `X`. )DOC"); } }; diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu index a21bc7335126ba8cff0de70aa8b8cc1fc36c1976..d837f49126e91bc937e346a829c204bd0bcd38fa 100644 --- a/paddle/operators/cross_entropy_op.cu +++ b/paddle/operators/cross_entropy_op.cu @@ -118,7 +118,7 @@ class CrossEntropyOpCUDAKernel : public framework::OpKernel { int grid = (n + block - 1) / block; // TODO(qingqing) launch kernel on specified stream // base on ExecutionContext. - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { auto* label_data = ctx.Input("Label")->data(); grid = d; SoftCrossEntropyKernel<<>>(y_data, x_data, @@ -154,7 +154,7 @@ class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { grid = (n + block - 1) / block; // TODO(qingqing): launch kernel on specified stream // base on ExecutionContext. - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { int block_x = 32; int block_y = 32; dim3 block(block_x, block_y); diff --git a/paddle/operators/cross_entropy_op.h b/paddle/operators/cross_entropy_op.h index 4bbd05a1bb677b3fd4f9dac9a19c0d2bcdb3185c..cfb00fe647fe2ecda5323e9d170715d852e8043c 100644 --- a/paddle/operators/cross_entropy_op.h +++ b/paddle/operators/cross_entropy_op.h @@ -53,7 +53,7 @@ class CrossEntropyOpKernel : public framework::OpKernel { y->mutable_data(ctx.GetPlace()); const int batch_size = x->dims()[0]; - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { auto prob = EigenMatrix::From(*x); auto lbl_mat = EigenMatrix::From(*labels); auto loss = EigenMatrix::From(*y); @@ -100,7 +100,7 @@ class CrossEntropyGradientOpKernel : public framework::OpKernel { int class_num = x->dims()[1]; // TODO(qingqing): make zero setting an common function. - if (ctx.Attr("soft_label") == 1) { + if (ctx.Attr("soft_label")) { auto* label_data = ctx.Input("Label")->data(); int index = 0; for (int i = 0; i < batch_size; ++i) { diff --git a/paddle/operators/detail/strided_memcpy.h b/paddle/operators/detail/strided_memcpy.h new file mode 100644 index 0000000000000000000000000000000000000000..b165224b37fb091c094a823179256c3dd40a37c9 --- /dev/null +++ b/paddle/operators/detail/strided_memcpy.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/ddim.h" +#include "paddle/memory/memcpy.h" +#include "paddle/platform/device_context.h" + +namespace paddle { +namespace operators { +namespace detail { + +template +struct StridedMemcpyFunctor; + +template +struct StridedMemcpyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim<1> src_stride, framework::Dim<1> dst_dim, + framework::Dim<1> dst_stride, T* dst) const { + auto place = dev_ctx.GetPlace(); + if (platform::is_cpu_place(place)) { + auto& cpu_place = boost::get(place); + memory::Copy(cpu_place, dst, cpu_place, src, sizeof(T) * dst_dim.head); + } else { +#ifndef PADDLE_ONLY_CPU + auto& gpu_place = boost::get(place); + auto& cuda_ctx = + reinterpret_cast(dev_ctx); + memory::Copy(gpu_place, dst, gpu_place, src, sizeof(T) * dst_dim.head, + cuda_ctx.stream()); +#else + PADDLE_THROW("Paddle is not compiled with GPU"); +#endif + } + } +}; + +template +struct StridedMemcpyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim src_stride, framework::Dim dst_dim, + framework::Dim dst_stride, T* dst) const { + for (int64_t i = 0; i < dst_dim.head; ++i) { + StridedMemcpyFunctor func; + func(dev_ctx, src, src_stride.tail, dst_dim.tail, dst_stride.tail, dst); + src += src_stride.head; + dst += dst_stride.head; + } + } +}; + +template +struct StridedCopyDimVisitor : public boost::static_visitor { + StridedCopyDimVisitor(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_stride, T* dst) + : dev_ctx_(dev_ctx), + src_(src), + src_stride_(src_stride), + dst_stride_(dst_stride), + dst_(dst) {} + + template + void operator()(Dim dst_dim) const { + Dim src_stride = boost::get(src_stride_); + Dim dst_stride = boost::get(dst_stride_); + constexpr int dim = Dim::dimensions; + StridedMemcpyFunctor functor; + functor(dev_ctx_, src_, src_stride, dst_dim, dst_stride, dst_); + } + + const platform::DeviceContext& dev_ctx_; + const T* src_; + const framework::DDim& src_stride_; + const framework::DDim& dst_stride_; + T* dst_; +}; + +} // namespace detail +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc index b111b9fccb2310bd5fb92bda878a497c51f62ce0..2130eda6a42c893d8ec251a7022a0bfa44433bb7 100644 --- a/paddle/operators/dropout_op.cc +++ b/paddle/operators/dropout_op.cc @@ -18,7 +18,6 @@ namespace paddle { namespace operators { using framework::Tensor; -using framework::LoDTensor; class DropoutOp : public framework::OperatorWithKernel { public: @@ -29,15 +28,13 @@ class DropoutOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); - // TODO(xinghai-sun): remove this check after swtiching to bool - PADDLE_ENFORCE(ctx.Attr("is_training") == 0 || - ctx.Attr("is_training") == 1); auto dims = ctx.Input("X")->dims(); - ctx.Output("Out")->Resize(dims); - if (ctx.Attr("is_training") == 1) { - ctx.Output("Mask")->Resize(dims); + ctx.Output("Out")->Resize(dims); + if (ctx.Attr("is_training")) { + ctx.Output("Mask")->Resize(dims); } + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -49,8 +46,7 @@ class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddAttr("dropout_prob", "Probability of setting units to zero.") .SetDefault(.5f); - // TODO(xinghai-sun): use bool for is_training after bool is supported. - AddAttr("is_training", "Whether in training phase.").SetDefault(1); + AddAttr("is_training", "Whether in training phase.").SetDefault(true); AddAttr("seed", "Dropout random seed.").SetDefault(0); AddInput("X", "The input of dropout op."); AddOutput("Out", "The output of dropout op."); @@ -59,7 +55,7 @@ class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Dropout Operator. -"Dropout" refers to randomly dropping out units in a nerual network. It is a +'Dropout' refers to randomly dropping out units in a nerual network. It is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly set (according to the given dropout probability) the outputs of some units to zero, while others @@ -75,8 +71,8 @@ class DropoutOpGrad : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_EQ(ctx.Attr("is_training"), 1, - "GradOp is only callable when is_training is true"); + PADDLE_ENFORCE(ctx.Attr("is_training"), + "GradOp is only callable when is_training is true"); PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Mask"), "Mask must not be null."); @@ -85,9 +81,6 @@ class DropoutOpGrad : public framework::OperatorWithKernel { PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); - // TODO(xinghai-sun): remove this check after swtiching to bool - PADDLE_ENFORCE(ctx.Attr("is_training") == 0 || - ctx.Attr("is_training") == 1); auto x_dims = ctx.Input("X")->dims(); auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); PADDLE_ENFORCE_EQ(x_dims, out_dims, @@ -96,7 +89,7 @@ class DropoutOpGrad : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(x_dims, mask_dims, "Dimensions of Input(X) and Mask must be the same."); - auto *x_grad = ctx.Output(framework::GradVarName("X")); + auto *x_grad = ctx.Output(framework::GradVarName("X")); x_grad->Resize(x_dims); } }; diff --git a/paddle/operators/dropout_op.cu b/paddle/operators/dropout_op.cu index 186237fb238add37f32403309a0f7e8a9846d335..a04e4a22cc09d4e8106a528e490ccf8e90681c08 100644 --- a/paddle/operators/dropout_op.cu +++ b/paddle/operators/dropout_op.cu @@ -59,7 +59,7 @@ class GPUDropoutKernel : public framework::OpKernel { auto Y = EigenMatrix::Reshape(*y, 1); auto place = context.GetEigenDevice(); - if (context.Attr("is_training") == 1) { + if (context.Attr("is_training")) { auto* mask = context.Output("Mask"); auto* mask_data = mask->mutable_data(context.GetPlace()); int size = framework::product(mask->dims()); diff --git a/paddle/operators/dropout_op.h b/paddle/operators/dropout_op.h index 82eafee0e0e7db7b4b4ae5405f37146d061aefd5..d57f64afcb3558aeea6aed23fae06866e9af874a 100644 --- a/paddle/operators/dropout_op.h +++ b/paddle/operators/dropout_op.h @@ -35,7 +35,7 @@ class CPUDropoutKernel : public framework::OpKernel { auto* y_data = y->mutable_data(context.GetPlace()); AttrType dropout_prob = context.Attr("dropout_prob"); - if (context.Attr("is_training") == 1) { + if (context.Attr("is_training")) { auto* mask = context.Output("Mask"); auto* mask_data = mask->mutable_data(context.GetPlace()); int seed = context.Attr("seed"); @@ -65,8 +65,8 @@ template class DropoutGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - PADDLE_ENFORCE_EQ(context.Attr("is_training"), 1, - "GradOp is only callable when is_training is true"); + PADDLE_ENFORCE(context.Attr("is_training"), + "GradOp is only callable when is_training is true"); auto* grad_x = context.Output(framework::GradVarName("X")); auto* grad_y = context.Input(framework::GradVarName("Out")); diff --git a/paddle/operators/elementwise_add_op.cc b/paddle/operators/elementwise_add_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..5f7b654d69f081dfa85b0d61960eb52b7982faa1 --- /dev/null +++ b/paddle/operators/elementwise_add_op.cc @@ -0,0 +1,39 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_add_op.h" + +namespace paddle { +namespace operators { +class ElementwiseAddOpMaker : public ElementwiseOpMaker { + public: + ElementwiseAddOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("add", "Out = X + Y"); + AddComment(comment_); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(elementwise_add, ops::ElementwiseOp, ops::ElementwiseAddOpMaker, + elementwise_add_grad, ops::ElementwiseOpGrad); +REGISTER_OP_CPU_KERNEL( + elementwise_add, + ops::ElementwiseAddKernel); +REGISTER_OP_CPU_KERNEL( + elementwise_add_grad, + ops::ElementwiseAddGradKernel); diff --git a/paddle/operators/elementwise_add_op.cu b/paddle/operators/elementwise_add_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..85d063a76b5592c716a5bdf23a0993976abc6ae4 --- /dev/null +++ b/paddle/operators/elementwise_add_op.cu @@ -0,0 +1,25 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/elementwise_add_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + elementwise_add, + ops::ElementwiseAddKernel); +REGISTER_OP_GPU_KERNEL( + elementwise_add_grad, + ops::ElementwiseAddGradKernel); diff --git a/paddle/operators/elementwise_add_op.h b/paddle/operators/elementwise_add_op.h new file mode 100644 index 0000000000000000000000000000000000000000..42a7a29e991adcce98c74c62bb359a2cc4a7c30d --- /dev/null +++ b/paddle/operators/elementwise_add_op.h @@ -0,0 +1,113 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_op.h" + +namespace paddle { +namespace operators { + +template +class ElementwiseAddKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseCompute(ctx); + } +}; + +template +struct ElementwiseAddGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = dz_e; + } + } +}; + +template +struct ElementwiseAddOneGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = dz_e.sum(); + } + } +}; + +template +struct ElementwiseAddBroadCastGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = dz_e.reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); + } + } +}; + +template +struct ElementwiseAddBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = dz_e.reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); + } + } +}; + +template +class ElementwiseAddGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseAddOneGradFunctor, + ElementwiseAddBroadCastGradFunctor, + ElementwiseAddBroadCast2GradFunctor>(ctx); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/elementwise_div_op.cc b/paddle/operators/elementwise_div_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..c6898150d310d0c4fdefae5a58a5792a72f9889e --- /dev/null +++ b/paddle/operators/elementwise_div_op.cc @@ -0,0 +1,40 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_div_op.h" + +namespace paddle { +namespace operators { +class ElementwiseDivOpMaker : public ElementwiseOpMaker { + public: + ElementwiseDivOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("Div", "Out = X / Y"); + AddComment(comment_); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(elementwise_div, ops::ElementwiseOp, ops::ElementwiseDivOpMaker, + elementwise_div_grad, ops::ElementwiseOpGrad); +REGISTER_OP_CPU_KERNEL( + elementwise_div, + ops::ElementwiseDivKernel); +REGISTER_OP_CPU_KERNEL( + elementwise_div_grad, + ops::ElementwiseDivGradKernel); diff --git a/paddle/operators/elementwise_div_op.cu b/paddle/operators/elementwise_div_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..b96aa31748c77f0d07f9bb7fb19235239983abd5 --- /dev/null +++ b/paddle/operators/elementwise_div_op.cu @@ -0,0 +1,25 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/elementwise_div_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + elementwise_div, + ops::ElementwiseDivKernel); +REGISTER_OP_GPU_KERNEL( + elementwise_div_grad, + ops::ElementwiseDivGradKernel); diff --git a/paddle/operators/elementwise_div_op.h b/paddle/operators/elementwise_div_op.h new file mode 100644 index 0000000000000000000000000000000000000000..6ef60cdf8dfc01f7cf51bb68f18900effb444fca --- /dev/null +++ b/paddle/operators/elementwise_div_op.h @@ -0,0 +1,115 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_op.h" + +namespace paddle { +namespace operators { + +template +class ElementwiseDivKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseCompute(ctx); + } +}; + +template +struct ElementwiseDivGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto y_e = framework::EigenVector::Flatten(*y); + auto z_e = framework::EigenVector::Flatten(*z); + auto dz_e = framework::EigenVector::Flatten(*dz); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e / y_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = -1.0 * dz_e * z_e / y_e; + } + } +}; + +template +struct ElementwiseDivBroadCastGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n)) + .broadcast(Eigen::DSizes(pre, 1)) + .reshape(Eigen::DSizes(x_e.size())); + + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e / y_e_bcast; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0 * (x_e * dz_e) / (y_e_bcast * y_e_bcast)) + .reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); + } + } +}; + +template +struct ElementwiseDivBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) + .broadcast(Eigen::DSizes(pre, 1, post)) + .reshape(Eigen::DSizes(x_e.size())); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e / y_e_bcast; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0 * (x_e * dz_e) / (y_e_bcast * y_e_bcast)) + .reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); + } + } +}; + +template +class ElementwiseDivGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseDivGradFunctor, + ElementwiseDivBroadCastGradFunctor, + ElementwiseDivBroadCast2GradFunctor>(ctx); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc index ee6e975b443691bf71cec904565ced20406f3fba..f2544b54d6bc543a50d8de03d482333b485bc076 100644 --- a/paddle/operators/elementwise_mul_op.cc +++ b/paddle/operators/elementwise_mul_op.cc @@ -17,101 +17,25 @@ namespace paddle { namespace operators { -using Tensor = framework::Tensor; - -class ElementWiseMulOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of ElementWiseMulOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Input(Y) of ElementWiseMulOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Out"), - "Output(Out) of ElementWiseMulOp should not be null."); - - auto x_dim = ctx.Input("X")->dims(); - auto y_dim = ctx.Input("Y")->dims(); - PADDLE_ENFORCE_GE(x_dim.size(), y_dim.size(), - "Rank of first input must >= rank of second input.") - ctx.Output("Out")->Resize(x_dim); - } -}; - -class ElementWiseMulOpMaker : public framework::OpProtoAndCheckerMaker { +class ElementwiseMulOpMaker : public ElementwiseOpMaker { public: - ElementWiseMulOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of elementwise mul op"); - AddInput("Y", "The second input of elementwise mul op"); - AddAttr("axis", - R"DOC( -When shape(Y) does not equal shape(X),Y will be broadcasted -to match the shape of X and axis should be dimension index Y in X - )DOC") - .SetDefault(-1) - .EqualGreaterThan(-1); - - AddOutput("Out", "The output of elementwise mul op"); - AddComment(R"DOC( -Limited elementwise multiple operator.The equation is: Out = X ⊙ Y. -1. The shape of Y should be same with X or -2. Y's shape is a subset of X. - Y will be broadcasted to match the shape of X and axis should be dimension index Y in X. - example: - shape(X) = (2, 3, 4, 5), shape(Y) = (,) - shape(X) = (2, 3, 4, 5), shape(Y) = (5,) - shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) - shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 - shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 -)DOC"); + ElementwiseMulOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("Mul", "Out = X ⊙ Y"); + AddComment(comment_); } }; -class ElementWiseMulOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) should not be null"); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), - "Input(Out@GRAD) should not be null"); - - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - auto *x_grad = - ctx.Output(framework::GradVarName("X")); - auto *y_grad = - ctx.Output(framework::GradVarName("Y")); - - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input.") - - if (x_grad) { - x_grad->Resize(x_dims); - } - - if (y_grad) { - y_grad->Resize(y_dims); - } - } -}; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(elementwise_mul, ops::ElementWiseMulOp, ops::ElementWiseMulOpMaker, - elementwise_mul_grad, ops::ElementWiseMulOpGrad); +REGISTER_OP(elementwise_mul, ops::ElementwiseOp, ops::ElementwiseMulOpMaker, + elementwise_mul_grad, ops::ElementwiseOpGrad); REGISTER_OP_CPU_KERNEL( elementwise_mul, - ops::ElementWiseMulKernel); + ops::ElementwiseMulKernel); REGISTER_OP_CPU_KERNEL( elementwise_mul_grad, - ops::ElementWiseMulGradKernel); + ops::ElementwiseMulGradKernel); diff --git a/paddle/operators/elementwise_mul_op.cu b/paddle/operators/elementwise_mul_op.cu index 56f2087c22c6c599a3c5aef36eb0fe3eac295bef..da08a75596c4d3b89dc8892bd4405464fec96389 100644 --- a/paddle/operators/elementwise_mul_op.cu +++ b/paddle/operators/elementwise_mul_op.cu @@ -19,7 +19,7 @@ namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL( elementwise_mul, - ops::ElementWiseMulKernel); + ops::ElementwiseMulKernel); REGISTER_OP_GPU_KERNEL( elementwise_mul_grad, - ops::ElementWiseMulGradKernel); + ops::ElementwiseMulGradKernel); diff --git a/paddle/operators/elementwise_mul_op.h b/paddle/operators/elementwise_mul_op.h index 6d58da580b81b9e0a8ae170eec1a73638b190df8..1eaf2e3efc97a32739efcaf37066817ee173fadc 100644 --- a/paddle/operators/elementwise_mul_op.h +++ b/paddle/operators/elementwise_mul_op.h @@ -13,171 +13,104 @@ limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" +#include "paddle/operators/elementwise_op.h" namespace paddle { namespace operators { -/* - * Out = X ⊙ Y - * 1. shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 - * pre=2, n=3*4, post=5 - * 2. shape(X) = (2, 3, 4, 5), shape(Y) = (4,5) - * pre=2*3, n=4*5, post=1 - */ - -inline void get_mid_dims(const framework::DDim& x_dims, - const framework::DDim& y_dims, const int axis, - int& pre, int& n, int& post) { - pre = 1; - n = 1; - post = 1; - for (int i = 0; i < axis; ++i) { - pre *= x_dims[i]; - } - - for (int i = 0; i < y_dims.size(); ++i) { - PADDLE_ENFORCE_EQ(x_dims[i + axis], y_dims[i], - "Broadcast dimension mismatch."); - n *= y_dims[i]; - } - - for (int i = axis + y_dims.size(); i < x_dims.size(); ++i) { - post *= x_dims[i]; - } -} template -class ElementWiseMulKernel : public framework::OpKernel { +class ElementwiseMulKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); + ElementwiseCompute(ctx); + } +}; +template +struct ElementwiseMulGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { auto x_e = framework::EigenVector::Flatten(*x); auto y_e = framework::EigenVector::Flatten(*y); - auto z_e = framework::EigenVector::Flatten(*z); + auto dz_e = framework::EigenVector::Flatten(*dz); - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input.") - - if (x_dims == y_dims || product(y_dims) == 1) { - z_e.device(ctx.GetEigenDevice()) = x_e * y_e; - return; + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e * y_e; } - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - auto y_bcast = y_e.reshape(Eigen::DSizes(1, n)) - .broadcast(Eigen::DSizes(pre, 1)) - .reshape(Eigen::DSizes(x_e.size())); - z_e.device(ctx.GetEigenDevice()) = x_e * y_bcast; - return; - } else { - auto y_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) - .broadcast(Eigen::DSizes(pre, 1, post)) - .reshape(Eigen::DSizes(x_e.size())); - z_e.device(ctx.GetEigenDevice()) = x_e * y_bcast; - return; + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = x_e * dz_e; } } }; -template -class ElementWiseMulGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* dout = ctx.Input(framework::GradVarName("Out")); - +template +struct ElementwiseMulBroadCastGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { auto x_e = framework::EigenVector::Flatten(*x); auto y_e = framework::EigenVector::Flatten(*y); - auto dout_e = framework::EigenVector::Flatten(*dout); + auto dz_e = framework::EigenVector::Flatten(*dz); - auto x_dims = x->dims(); - auto y_dims = y->dims(); + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n)) + .broadcast(Eigen::DSizes(pre, 1)) + .reshape(Eigen::DSizes(x_e.size())); - auto* dx = ctx.Output(framework::GradVarName("X")); - auto* dy = ctx.Output(framework::GradVarName("Y")); if (dx) { - dx->mutable_data(ctx.GetPlace()); + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e * y_e_bcast; } + if (dy) { - dy->mutable_data(ctx.GetPlace()); + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (x_e * dz_e) + .reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); } + } +}; - if (x_dims == y_dims || product(y_dims) == 1) { - if (dx) { - auto dx_e = framework::EigenVector::Flatten(*dx); - dx_e.device(ctx.GetEigenDevice()) = dout_e * y_e; - } - - if (dy) { - auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(ctx.GetEigenDevice()) = x_e * dout_e; - } - return; +template +struct ElementwiseMulBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { + auto x_e = framework::EigenVector::Flatten(*x); + auto y_e = framework::EigenVector::Flatten(*y); + auto dz_e = framework::EigenVector::Flatten(*dz); + + auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) + .broadcast(Eigen::DSizes(pre, 1, post)) + .reshape(Eigen::DSizes(x_e.size())); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e * y_e_bcast; } - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - - // TODO(gongweibao): wrap reshape to a function. - if (post == 1) { - auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n)) - .broadcast(Eigen::DSizes(pre, 1)) - .reshape(Eigen::DSizes(x_e.size())); - if (dx) { - auto dx_e = framework::EigenVector::Flatten(*dx); - dx_e.device(ctx.GetEigenDevice()) = dout_e * y_e_bcast; - } - - if (dy) { - auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(ctx.GetEigenDevice()) = - (x_e * dout_e) - .reshape(Eigen::DSizes(pre, n)) - .sum(Eigen::array{{0}}); - } - return; - } else { - auto y_e_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) - .broadcast(Eigen::DSizes(pre, 1, post)) - .reshape(Eigen::DSizes(x_e.size())); - if (dx) { - auto dx_e = framework::EigenVector::Flatten(*dx); - dx_e.device(ctx.GetEigenDevice()) = dout_e * y_e_bcast; - } - - if (dy) { - auto dy_e = framework::EigenVector::Flatten(*dy); - dy_e.device(ctx.GetEigenDevice()) = - (x_e * dout_e) - .reshape(Eigen::DSizes(pre, n, post)) - .sum(Eigen::array{{0, 2}}); - } - return; + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (x_e * dz_e) + .reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); } } }; +template +class ElementwiseMulGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseMulGradFunctor, + ElementwiseMulBroadCastGradFunctor, + ElementwiseMulBroadCast2GradFunctor>(ctx); + } +}; + } // namespace operators } // namespace paddle diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h new file mode 100644 index 0000000000000000000000000000000000000000..f224722c1bec6716e68de9da2509250f7d4b37ae --- /dev/null +++ b/paddle/operators/elementwise_op.h @@ -0,0 +1,312 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +/* + * Out = X ⊙ Y + * If Y's shape does not match X' shape, they will be reshaped. + * For example: + * 1. shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 + * pre=2, n=3*4, post=5 + * x.shape(2, 12, 5) * y.shape(1,12,1).broadcast(2,12,5) + * 2. shape(X) = (2, 3, 4, 5), shape(Y) = (4,5) + * pre=2*3, n=4*5, post=1 + * x.shape(2, 3, 20) * y.shape(1,1,20).broadcast(2,3,20) + */ +inline void get_mid_dims(const framework::DDim& x_dims, + const framework::DDim& y_dims, const int axis, + int& pre, int& n, int& post) { + pre = 1; + n = 1; + post = 1; + for (int i = 0; i < axis; ++i) { + pre *= x_dims[i]; + } + + for (int i = 0; i < y_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(x_dims[i + axis], y_dims[i], + "Broadcast dimension mismatch."); + n *= y_dims[i]; + } + + for (int i = axis + y_dims.size(); i < x_dims.size(); ++i) { + post *= x_dims[i]; + } +} + +#define EIGEN_FUNCTOR(name, eigen_op) \ + struct Eigen##name##Functor { \ + template \ + inline void Run(const framework::Tensor* x, const framework::Tensor* y, \ + framework::Tensor* z, \ + const framework::ExecutionContext& ctx) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_e); \ + } \ + template \ + inline void RunBroadCast(const framework::Tensor* x, \ + const framework::Tensor* y, framework::Tensor* z, \ + const framework::ExecutionContext& ctx, int pre, \ + int n) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + auto y_bcast = y_e.reshape(Eigen::DSizes(1, n)) \ + .broadcast(Eigen::DSizes(pre, 1)) \ + .reshape(Eigen::DSizes(x_e.size())); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + } \ + template \ + inline void RunBroadCast2(const framework::Tensor* x, \ + const framework::Tensor* y, \ + framework::Tensor* z, \ + const framework::ExecutionContext& ctx, int pre, \ + int n, int post) { \ + auto x_e = framework::EigenVector::Flatten(*x); \ + auto y_e = framework::EigenVector::Flatten(*y); \ + auto z_e = framework::EigenVector::Flatten(*z); \ + auto y_bcast = y_e.reshape(Eigen::DSizes(1, n, 1)) \ + .broadcast(Eigen::DSizes(pre, 1, post)) \ + .reshape(Eigen::DSizes(x_e.size())); \ + z_e.device(ctx.GetEigenDevice()) = eigen_op(x_e, y_bcast); \ + } \ + } + +template +void ElementwiseCompute(const framework::ExecutionContext& ctx) { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* z = ctx.Output("Out"); + z->mutable_data(ctx.GetPlace()); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input.") + + if (x_dims == y_dims || product(y_dims) == 1) { + functor f; + f.template Run(x, y, z, ctx); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), + "Axis should be in range [0, x_dims)"); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + if (post == 1) { + functor f; + f.template RunBroadCast(x, y, z, ctx, pre, n); + return; + } else { + functor f; + f.template RunBroadCast2(x, y, z, ctx, pre, n, post); + return; + } +} + +#define EIGEN_ADD(x, y) ((x) + (y)) +EIGEN_FUNCTOR(Add, EIGEN_ADD); + +#define EIGEN_SUB(x, y) ((x) - (y)) +EIGEN_FUNCTOR(Sub, EIGEN_SUB); + +#define EIGEN_MUL(x, y) ((x) * (y)) +EIGEN_FUNCTOR(Mul, EIGEN_MUL); + +#define EIGEN_DIV(x, y) ((x) / (y)) +EIGEN_FUNCTOR(Div, EIGEN_DIV); + +template +void ElementwiseGradCompute(const framework::ExecutionContext& ctx) { + using Tensor = framework::Tensor; + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* out = ctx.Input("Out"); + auto* dout = ctx.Input(framework::GradVarName("Out")); + + auto place = ctx.GetEigenDevice(); + + auto x_dims = x->dims(); + auto y_dims = y->dims(); + + auto* dx = ctx.Output(framework::GradVarName("X")); + auto* dy = ctx.Output(framework::GradVarName("Y")); + if (dx) { + dx->mutable_data(ctx.GetPlace()); + } + if (dy) { + dy->mutable_data(ctx.GetPlace()); + } + + if (x_dims == y_dims) { + functor f; + f(place, x, y, out, dx, dy, dout); + return; + } + + if (product(y_dims) == 1) { + functor1 f; + f(place, x, y, out, dx, dy, dout); + return; + } + + int axis = ctx.Attr("axis"); + axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + + int pre, n, post; + get_mid_dims(x_dims, y_dims, axis, pre, n, post); + + if (post == 1) { + broadcastfunctor f; + f(place, x, y, out, dx, dy, dout, pre, n); + return; + } else { + broadcast2functor f; + f(place, x, y, out, dx, dy, dout, pre, n, post); + return; + } +} + +class ElementwiseOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + using Tensor = framework::Tensor; + void InferShape(const framework::InferShapeContext& ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of elementwise op should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), + "Input(Y) of elementwise op should not be null"); + PADDLE_ENFORCE_NOT_NULL( + ctx.OutputVar("Out"), + "Output(Out) of elementwise op should not be null."); + + auto x_dim = ctx.Input("X")->dims(); + auto y_dim = ctx.Input("Y")->dims(); + PADDLE_ENFORCE_GE(x_dim.size(), y_dim.size(), + "Rank of first input must >= rank of second input.") + ctx.Output("Out")->Resize(x_dim); + ctx.ShareLoD("X", /*->*/ "Out"); + } +}; + +class ElementwiseOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ElementwiseOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", R"DOC( +The first input of elementwise op, it's a tensor of any dimensions. +)DOC"); + AddInput("Y", R"DOC( +The sencond input of elementwise op, it's a tensor and it's dimensions +must be small or equal to X's dimensions. +)DOC"); + AddAttr("axis", + R"DOC( +When the shape(Y) does not equal the shape(X),Y will be broadcasted +to match the shape of X and axis should be dimension index Y in X + )DOC") + .SetDefault(-1) + .EqualGreaterThan(-1); + + AddOutput("Out", "The output of elementwise op"); + comment_ = R"DOC( +Limited elementwise {name} operator.The equation is: Out = {equation}. +1. The shape of Y should be same with X or +2. Y's shape is a subset of X. + Y will be broadcasted to match the shape of X and axis should be dimension index Y in X. + + example: + shape(X) = (2, 3, 4, 5), shape(Y) = (,) + shape(X) = (2, 3, 4, 5), shape(Y) = (5,) + shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) + shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 + shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 + +Both the input X and Y can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with input X. +)DOC"; + AddComment(comment_); + } + + protected: + std::string comment_; + + void Replace(std::string& src, std::string from, std::string to) { + std::size_t len_from = std::strlen(from.c_str()); + std::size_t len_to = std::strlen(to.c_str()); + for (std::size_t pos = src.find(from); pos != std::string::npos; + pos = src.find(from, pos + len_to)) { + src.replace(pos, len_from, to); + } + } + + void SetComment(std::string name, std::string equation) { + Replace(comment_, "{name}", name); + Replace(comment_, "{equation}", equation); + } +}; + +class ElementwiseOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + using Tensor = framework::Tensor; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Input(Y) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + + auto x_dims = ctx.Input("X")->dims(); + auto y_dims = ctx.Input("Y")->dims(); + auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); + auto* x_grad = ctx.Output(framework::GradVarName("X")); + auto* y_grad = ctx.Output(framework::GradVarName("Y")); + + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input.") + + if (x_grad) { + x_grad->Resize(x_dims); + } + + if (y_grad) { + y_grad->Resize(y_dims); + } + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/elementwise_sub_op.cc b/paddle/operators/elementwise_sub_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..31c37ff7ab5595c29f973929387d3945b6f3aaf8 --- /dev/null +++ b/paddle/operators/elementwise_sub_op.cc @@ -0,0 +1,39 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_sub_op.h" + +namespace paddle { +namespace operators { +class ElementwiseSubOpMaker : public ElementwiseOpMaker { + public: + ElementwiseSubOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : ElementwiseOpMaker(proto, op_checker) { + SetComment("Sub", "Out = X - Y"); + AddComment(comment_); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(elementwise_sub, ops::ElementwiseOp, ops::ElementwiseSubOpMaker, + elementwise_sub_grad, ops::ElementwiseOpGrad); +REGISTER_OP_CPU_KERNEL( + elementwise_sub, + ops::ElementwiseSubKernel); +REGISTER_OP_CPU_KERNEL( + elementwise_sub_grad, + ops::ElementwiseSubGradKernel); diff --git a/paddle/operators/elementwise_sub_op.cu b/paddle/operators/elementwise_sub_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..0efb92fce9975ed9fa029a3ce919589d09efb0d7 --- /dev/null +++ b/paddle/operators/elementwise_sub_op.cu @@ -0,0 +1,25 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/elementwise_sub_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + elementwise_sub, + ops::ElementwiseSubKernel); +REGISTER_OP_GPU_KERNEL( + elementwise_sub_grad, + ops::ElementwiseSubGradKernel); diff --git a/paddle/operators/elementwise_sub_op.h b/paddle/operators/elementwise_sub_op.h new file mode 100644 index 0000000000000000000000000000000000000000..faa38cf4014e98b2a28f51b32585ca5c10289841 --- /dev/null +++ b/paddle/operators/elementwise_sub_op.h @@ -0,0 +1,115 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/elementwise_op.h" + +namespace paddle { +namespace operators { + +template +class ElementwiseSubKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseCompute(ctx); + } +}; + +template +struct ElementwiseSubGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0) * dz_e; + } + } +}; + +template +struct ElementwiseSubOneGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0) * dz_e.sum(); + } + } +}; + +template +struct ElementwiseSubBroadCastGradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0) * + dz_e.reshape(Eigen::DSizes(pre, n)) + .sum(Eigen::array{{0}}); + } + } +}; + +template +struct ElementwiseSubBroadCast2GradFunctor { + template + void operator()(Device d, X x, Y y, Z z, dX dx, dY dy, dZ dz, Pre pre, N n, + Post post) { + auto dz_e = framework::EigenVector::Flatten(*dz); + if (dx) { + auto dx_e = framework::EigenVector::Flatten(*dx); + dx_e.device(d) = dz_e; + } + + if (dy) { + auto dy_e = framework::EigenVector::Flatten(*dy); + dy_e.device(d) = (-1.0) * + dz_e.reshape(Eigen::DSizes(pre, n, post)) + .sum(Eigen::array{{0, 2}}); + } + } +}; + +template +class ElementwiseSubGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + ElementwiseGradCompute, + ElementwiseSubOneGradFunctor, + ElementwiseSubBroadCastGradFunctor, + ElementwiseSubBroadCast2GradFunctor>(ctx); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/fc_op.cc b/paddle/operators/fc_op.cc index e5d0f3c3724262a60a463ef3beadd9906d3ebaf6..5ac0e8cc45f007d42f1b6d7f86333f5cbedb3ea8 100644 --- a/paddle/operators/fc_op.cc +++ b/paddle/operators/fc_op.cc @@ -186,6 +186,9 @@ W_i is a 2-D matrix of size (K x N), where N means the number of neurons in the fully connected layer. B is a 1-D vector of size N. Thus, the output Out is a 2-D matrix of size (M x N). Activation type can be set to `identity` (default), `sigmoid` or `softmax`. + +All the inputs can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with first input (`X[0]`). )DOC"); } }; diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index ba7857cc65f6860a6156674c6addc2bfdce21a99..761a527a5574edc779340ec595dfe1bc1964438a 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -23,15 +23,14 @@ class FillZerosLikeOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("Src"), - "Input(Src) of FillZerosLikeOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Dst"), - "Output(Dst) of FillZerosLikeOp should not be null."); - - ctx.Output("Dst")->Resize( - ctx.Input("Src")->dims()); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of FillZerosLikeOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), + "Output(Y) of FillZerosLikeOp should not be null."); + + ctx.Output("Y")->Resize( + ctx.Input("X")->dims()); + ctx.ShareLoD("X", /*->*/ "Y"); } }; @@ -40,8 +39,8 @@ class FillZerosLikeOpMaker : public framework::OpProtoAndCheckerMaker { FillZerosLikeOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Src", "The input of fill-zeros-like op."); - AddOutput("Dst", "The varibale will be filled up with zeros."); + AddInput("X", "The input of fill-zeros-like op."); + AddOutput("Y", "The varibale will be filled up with zeros."); AddComment(R"DOC( Fill up a vriable with zeros. diff --git a/paddle/operators/fill_zeros_like_op.h b/paddle/operators/fill_zeros_like_op.h index 969998ce2eae02b8ad057c6259703e51559bf98a..4474581784531faee1741f0b143743e31cc3788f 100644 --- a/paddle/operators/fill_zeros_like_op.h +++ b/paddle/operators/fill_zeros_like_op.h @@ -23,7 +23,7 @@ template class FillZerosLikeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* output = context.Output("Dst"); + auto* output = context.Output("Y"); output->mutable_data(context.GetPlace()); auto t = framework::EigenVector::Flatten(*output); t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc index d445b61c1657356f2cdcf1e98d756607de2bd042..fecd1ce2147a1e6f2f7928266be74ed7b647c5b9 100644 --- a/paddle/operators/gather_op.cc +++ b/paddle/operators/gather_op.cc @@ -35,7 +35,7 @@ class GatherOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_GE(batch_size, 0, "Batch size must be >0"); framework::DDim output_dims(ctx.Input("X")->dims()); output_dims[0] = batch_size; - ctx.Output("Out")->Resize(output_dims); + ctx.Output("Out")->Resize(output_dims); } }; @@ -45,7 +45,7 @@ class GatherGradOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - auto X_grad = ctx.Output(framework::GradVarName("X")); + auto X_grad = ctx.Output(framework::GradVarName("X")); auto X = ctx.Input("X"); X_grad->Resize(X->dims()); diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index c0e161bbc0c5486eb10408e43e6388f1b287abf8..5b7cbb5cc7bcb7e43b15363d37d7b8f2cbf0fbdc 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -48,7 +48,7 @@ class GaussianRandomOp : public framework::OperatorWithKernel { ctx.OutputVar("Out"), "Output(Out) of GaussianRandomOp should not be null."); - auto* tensor = ctx.Output("Out"); + auto* tensor = ctx.Output("Out"); auto dims = Attr>("dims"); std::vector temp; temp.reserve(dims.size()); diff --git a/paddle/operators/gemm_conv2d_op.h b/paddle/operators/gemm_conv2d_op.h new file mode 100644 index 0000000000000000000000000000000000000000..5c9e81732aa72211c2021382cf9a907880c53c17 --- /dev/null +++ b/paddle/operators/gemm_conv2d_op.h @@ -0,0 +1,226 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/im2col.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class GemmConv2DKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + // The filter will be reshaped in the calculations, + // so here use an assignment operation, + // that avoids modifying the variable in the Scope. + Tensor filter = *context.Input("Filter"); + Tensor* output = context.Output("Output"); + output->mutable_data(context.GetPlace()); + + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + int groups = context.Attr("groups"); + + int batch_size = input->dims()[0]; + int input_channels = input->dims()[1]; + int filter_height = filter.dims()[filter.dims().size() - 2]; + int filter_width = filter.dims()[filter.dims().size() - 1]; + int output_channels = output->dims()[1]; + int output_height = output->dims()[2]; + int output_width = output->dims()[3]; + + paddle::operators::math::Im2ColFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + im2col; + // use col_shape in the im2col calculation + framework::DDim col_shape = {input_channels / groups, filter_height, + filter_width, output_height, output_width}; + // use col_matrix_shape in the gemm calculation + framework::DDim col_matrix_shape = { + input_channels / groups * filter_height * filter_width, + output_height * output_width}; + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix = col; + col_matrix.Resize(col_matrix_shape); + + framework::DDim input_shape = {input->dims()[1], input->dims()[2], + input->dims()[3]}; + framework::DDim filter_matrix_shape = {filter.dims()[0], + filter.numel() / filter.dims()[0]}; + filter.Resize(filter_matrix_shape); + + framework::DDim output_matrix_shape = {output_channels, + output_height * output_width}; + + // convolution operator: im2col + gemm + int in_step = input_channels / groups; + int out_step = output_channels / groups; + for (int i = 0; i < batch_size; i++) { + Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); + Tensor out_batch = output->Slice(i, i + 1).Resize(output_matrix_shape); + for (int g = 0; g < groups; g++) { + // im2col + Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); + im2col(context.device_context(), in_slice, col, strides[0], strides[1], + paddings[0], paddings[1]); + + // gemm + Tensor out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor filter_slice = filter.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), filter_slice, false, + col_matrix, false, T(1.0), &out_slice, T(0.0)); + } + } + } +}; + +template +class GemmConvGrad2DKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + const Tensor* output_grad = + context.Input(framework::GradVarName("Output")); + Tensor* input_grad = + context.Output(framework::GradVarName("Input")); + Tensor* filter_grad = + context.Output(framework::GradVarName("Filter")); + + // The filter and filter_grad will be reshaped in the calculations, + // so here use an assignment operation, + // that avoids modifying the variable in the Scope. + Tensor filter = *context.Input("Filter"); + + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + int groups = context.Attr("groups"); + + int batch_size = input->dims()[0]; + int input_channels = input->dims()[1]; + int filter_height = filter.dims()[filter.dims().size() - 2]; + int filter_width = filter.dims()[filter.dims().size() - 1]; + int output_channels = output_grad->dims()[1]; + int output_height = output_grad->dims()[2]; + int output_width = output_grad->dims()[3]; + + paddle::operators::math::Col2ImFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + col2im; + paddle::operators::math::Im2ColFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + im2col; + // use col_shape in the im2col and col2im calculation + framework::DDim col_shape = {input_channels / groups, filter_height, + filter_width, output_height, output_width}; + // use col_matrix_shape in the gemm calculation + framework::DDim col_matrix_shape = { + input_channels / groups * filter_height * filter_width, + output_height * output_width}; + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix = col; + col_matrix.Resize(col_matrix_shape); + + framework::DDim input_shape = {input->dims()[1], input->dims()[2], + input->dims()[3]}; + framework::DDim output_matrix_shape = { + output_grad->dims()[1], + output_grad->dims()[2] * output_grad->dims()[3]}; + + framework::DDim filter_matrix_shape = {filter.dims()[0], + filter.numel() / filter.dims()[0]}; + filter.Resize(filter_matrix_shape); + + // convolution backward input operator: gemm + col2im + // convolution backward weight operator: im2col + gemm + int in_step = input_channels / groups; + int out_step = output_channels / groups; + + if (input_grad) { + input_grad->mutable_data(context.GetPlace()); + auto t = framework::EigenVector::Flatten(*input_grad); + t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + + for (int i = 0; i < batch_size; i++) { + Tensor out_grad_batch = + output_grad->Slice(i, i + 1).Resize(output_matrix_shape); + Tensor in_grad_batch = + input_grad->Slice(i, i + 1).Resize(input_shape); + for (int g = 0; g < groups; g++) { + // gemm + Tensor out_grad_slice = + out_grad_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor filter_slice = + filter.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), filter_slice, true, + out_grad_slice, false, T(1.0), &col_matrix, + T(0.0)); + + // col2im + Tensor in_grad_slice = + in_grad_batch.Slice(g * in_step, (g + 1) * in_step); + col2im(context.device_context(), in_grad_slice, col, strides[0], + strides[1], paddings[0], paddings[1]); + } + } + } + + if (filter_grad) { + filter_grad->mutable_data(context.GetPlace()); + Tensor filter_grad_ = *filter_grad; + filter_grad_.Resize(filter_matrix_shape); + auto t = framework::EigenVector::Flatten(filter_grad_); + t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + + for (int i = 0; i < batch_size; i++) { + Tensor out_grad_batch = + output_grad->Slice(i, i + 1).Resize(output_matrix_shape); + Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); + for (int g = 0; g < groups; g++) { + // im2col + Tensor out_grad_slice = + out_grad_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); + im2col(context.device_context(), in_slice, col, strides[0], + strides[1], paddings[0], paddings[1]); + + // gemm + Tensor filter_grad_slice = + filter_grad_.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), out_grad_slice, + false, col_matrix, true, T(1.0), + &filter_grad_slice, T(1.0)); + } + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc index 07f6dfabca5879e3de6004e59d2e87f7fa68d66c..04ac24662e9cfec6a49cd213cb76bdebc7b730c8 100644 --- a/paddle/operators/lookup_table_op.cc +++ b/paddle/operators/lookup_table_op.cc @@ -32,9 +32,10 @@ class LookupTableOp : public framework::OperatorWithKernel { auto table_t = ctx.Input("W"); auto ids_t = ctx.Input("Ids"); - auto output_t = ctx.Output("Out"); + auto output_t = ctx.Output("Out"); output_t->Resize({ids_t->dims()[0], table_t->dims()[1]}); + ctx.ShareLoD("Ids", /*->*/ "Out"); } }; @@ -50,9 +51,13 @@ class LookupTableOpMaker : public framework::OpProtoAndCheckerMaker { "An input with type int32 or int64" "contains the ids to be looked up in W."); AddOutput("Out", "The lookup results, which have the same type with W."); - AddComment( - "This operator is used to perform lookups on the parameter W," - "then concatenated into a dense tensor."); + AddComment(R"DOC( +This operator is used to perform lookups on the parameter W, +then concatenated into a dense tensor. + +The input `Ids` can carry the LoD (Level of Details) information, +or not. And the output only shares the LoD with input `Ids`. +)DOC"); } }; @@ -64,7 +69,7 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { void InferShape(const framework::InferShapeContext &context) const override { auto table = context.Input("W"); auto d_table = - context.Output(framework::GradVarName("W")); + context.Output(framework::GradVarName("W")); d_table->Resize(table->dims()); } }; diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index 5727c1cab16c1379ffe77f5594c057e93a042785..c08a3380f042886cd400df0d840e61856274619c 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -27,9 +27,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -79,9 +80,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -137,9 +138,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -197,9 +199,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; diff --git a/paddle/operators/math/im2col.cu b/paddle/operators/math/im2col.cu index 9bff7bee3c95093852305d392af0949b831e5665..01f60bfe70f844fdcfd5aa481c27d9f12ec51305 100644 --- a/paddle/operators/math/im2col.cu +++ b/paddle/operators/math/im2col.cu @@ -64,9 +64,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -84,9 +85,9 @@ class Im2ColFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + im2col<<(context) + .stream()>>>( im.data(), num_outputs, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width, col.data()); @@ -149,9 +150,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -174,9 +175,9 @@ class Col2ImFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + col2im<<(context) + .stream()>>>( num_kernels, col.data(), input_height + 2 * padding_height, input_width + 2 * padding_width, input_channels, filter_height, filter_width, stride_height, stride_width, padding_height, @@ -235,9 +236,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -268,9 +270,9 @@ class Im2ColFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + im2colOCF<<(context) + .stream()>>>( im.data(), col.data(), input_channels, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width); @@ -318,9 +320,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -351,9 +353,9 @@ class Col2ImFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + col2imOCF<<(context) + .stream()>>>( im.data(), col.data(), input_channels, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width); diff --git a/paddle/operators/math/im2col.h b/paddle/operators/math/im2col.h index 8958c5457cc2c3034c34ca82fb2e98cc06be63c5..7b717e1603c94cd77c74cb0d86f1d23e2692f9d8 100644 --- a/paddle/operators/math/im2col.h +++ b/paddle/operators/math/im2col.h @@ -72,17 +72,18 @@ enum class ColFormat { kCFO = 0, kOCF = 1 }; template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context); + int padding_width); }; template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context); + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width); }; } // namespace math diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc index 4f380388b108dc173d847f027ba5c9db387a87f8..f0b8c885918afe7f80edc465c6d9be7c11ac066f 100644 --- a/paddle/operators/math/im2col_test.cc +++ b/paddle/operators/math/im2col_test.cc @@ -78,8 +78,8 @@ void testIm2col() { PADDLE_THROW("no GPU support"); #endif // PADDLE_ONLY_CPU } - im2col(input, output_cfo, stride, stride, padding, padding, context); - im2col_ocf(input, output_ocf, stride, stride, padding, padding, context); + im2col(*context, input, output_cfo, stride, stride, padding, padding); + im2col_ocf(*context, input, output_ocf, stride, stride, padding, padding); float* out_cfo_ptr; if (paddle::platform::is_cpu_place(*place)) { diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 7d7eeb59a23435036dc33c1e4fe6dd1c4a1a2f62..b04384bda81b93f5db0be3206eee10ad5e854540 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -27,7 +27,7 @@ class MeanOp : public framework::OperatorWithKernel { "Input(X) of MeanOp should not be null."); PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), "Output(Out) of MeanOp should not be null."); - ctx.Output("Out")->Resize({1}); + ctx.Output("Out")->Resize({1}); } }; @@ -37,7 +37,8 @@ class MeanOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of mean op"); AddOutput("Out", "The output of mean op").NotInGradient(); - AddComment("Mean Operator"); + AddComment(R"DOC( Mean Operator +)DOC"); } }; @@ -47,7 +48,7 @@ class MeanGradOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) + ctx.Output(framework::GradVarName("X")) ->Resize(ctx.Input("X")->dims()); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index a97bbecdca1779df330d1053cf359bb658aa75c2..29cb85489bd05f6c1e7143d962eac0af26e75825 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -40,7 +40,8 @@ class MinusOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( left_tensor->numel(), right_tensor->numel(), "Minus operator must take two tensor with same num of elements"); - ctx.Output("Out")->Resize(left_tensor->dims()); + ctx.Output("Out")->Resize(left_tensor->dims()); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -54,7 +55,12 @@ class MinusOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC(Minus Operator -Equation: Out = X - Y +Equation: + + Out = X - Y + +Both the input `X` and `Y` can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with input `X`. )DOC"); } }; diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..8606c0d1e1bf7a52299528d30af0367d9f93edd2 --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cc @@ -0,0 +1,114 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/modified_huber_loss_op.h" + +namespace paddle { +namespace operators { + +class ModifiedHuberLossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + PADDLE_ENFORCE_NOT_NULL(context.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(context.InputVar("Y"), "Y must be initialized."); + + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "The shape of X and Y must be the same."); + PADDLE_ENFORCE_EQ(x->dims().size(), 2, "The tensor rank of X must be 2."); + PADDLE_ENFORCE_EQ(x->dims()[1], 1, "The 2nd dimension of X must be 1."); + + context.Output("IntermediateVal")->Resize(x->dims()); + context.Output("Out")->Resize({x->dims()[0], 1}); + } +}; + +class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ModifiedHuberLossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input tensor of modified huber loss op." + "X is 2-D tensor with shape [batch_size, 1]."); + AddInput("Y", + "The target labels of modified huber loss op." + "The shape of Y is same as X. Values of Y must be 0 or 1."); + AddOutput("IntermediateVal", + "Variable to save intermediate result which will be reused in " + "backward processing.") + .AsIntermediate(); + AddOutput("Out", "Classification loss for X."); + AddComment(R"DOC( +Modified huber loss is used in binary classification problem. The shape of +input X and target Y are both [N, 1] and so is the shape of output loss. +Since target Y is not differentiable, cacluating gradient for Y is illegal. +The formulation of modified huber loss is: + +L(y, f(x)) = max(0, 1 - yf(x))^2 for yf(x) >= -1, + -4yf(x) otherwise. + +Make sure the values of target label Y are in {0, 1} here. The operator will +scale values of Y to {-1, +1} when computing losses and gradients. +)DOC"); + } +}; + +class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + auto* intermediate_val = context.Input("IntermediateVal"); + auto* out_grad = context.Input(framework::GradVarName("Out")); + auto* x_grad = + context.Output(framework::GradVarName("X")); + + PADDLE_ENFORCE_NOT_NULL(x, "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(y, "Y must be initialized."); + PADDLE_ENFORCE_NOT_NULL(intermediate_val, + "Intermediate value must not be null."); + PADDLE_ENFORCE_NOT_NULL(out_grad, "Input(Out@Grad) must not be null."); + + PADDLE_ENFORCE_EQ( + intermediate_val->dims(), x->dims(), + "The shape of X and intermediate value must be the same."); + PADDLE_ENFORCE_EQ(out_grad->dims(), x->dims(), + "The shape of Input(Out@Grad) and X must be the same."); + + if (x_grad) x_grad->Resize(x->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(modified_huber_loss, ops::ModifiedHuberLossOp, + ops::ModifiedHuberLossOpMaker, modified_huber_loss_grad, + ops::ModifiedHuberLossGradOp); + +REGISTER_OP_CPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_CPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradCPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.cu b/paddle/operators/modified_huber_loss_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..bce760f95e72cfec05b07591e0fa1250168b112f --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cu @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include +#include +#include +#include "paddle/framework/op_registry.h" +#include "paddle/operators/modified_huber_loss_op.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +struct ModifiedHuberLossBackward { + template + HOSTDEVICE void operator()(Tuple t) const { + auto inter_val = thrust::get<1>(t); + auto y_val = thrust::get<2>(t); + auto out_grad = thrust::get<3>(t); + if (inter_val < -1) { + thrust::get<0>(t) = -4 * (2 * y_val - 1) * out_grad; + } else if (inter_val < 1) { + thrust::get<0>(t) = -2 * (1 - inter_val) * (2 * y_val - 1) * out_grad; + } else { + thrust::get<0>(t) = 0; + } + } +}; + +template +class ModifiedHuberLossGradGPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("IntermediateVal"); + auto* in2 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + + if (out0) { + auto counts = framework::product(in1->dims()); + auto y_ptr = thrust::device_pointer_cast(in0->data()); + auto inter_val_ptr = thrust::device_pointer_cast(in1->data()); + auto out_grad_ptr = thrust::device_pointer_cast(in2->data()); + thrust::device_ptr x_grad_ptr( + out0->mutable_data(context.GetPlace())); + + auto iter_begin = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr, inter_val_ptr, y_ptr, out_grad_ptr)); + + auto iter_end = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr + counts, inter_val_ptr + counts, + y_ptr + counts, out_grad_ptr + counts)); + + thrust::for_each(iter_begin, iter_end, ModifiedHuberLossBackward()); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_GPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradGPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..cb51007749e3c59572d4852959f4119ac377decc --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.h @@ -0,0 +1,105 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +struct CheckLabelValue { + HOSTDEVICE T operator()(const T& val) const { + PADDLE_ASSERT(val == static_cast(0) || val == static_cast(1)); + } +}; + +template +struct ModifiedHuberLossForward { + HOSTDEVICE T operator()(const T& val) const { + if (val < -1) { + return -4 * val; + } else if (val < 1) { + return (1 - val) * (1 - val); + } else { + return static_cast(0); + } + } +}; + +template +class ModifiedHuberLossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* out0 = context.Output("IntermediateVal"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + // make sure value's of Y in {0, 1} + y.unaryExpr(CheckLabelValue()); + auto inter_val = EigenVector::Flatten(*out0); + // scale y to {-1, +1} and compute x * y + inter_val.device(place) = x * (2 * y - static_cast(1)); + auto loss = EigenVector::Flatten(*out1); + loss.device(place) = inter_val.unaryExpr(ModifiedHuberLossForward()); + } +}; + +// CPU backward kernel +template +class ModifiedHuberLossGradCPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("IntermediateVal"); + auto* in2 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + + if (out0) { + const T* y_ptr = in0->data(); + const T* inter_val_ptr = in1->data(); + const T* out_grad_ptr = in2->data(); + size_t counts = static_cast(framework::product(in1->dims())); + T* x_grad_ptr = out0->mutable_data(context.GetPlace()); + for (size_t i = 0; i < counts; ++i) { + if (inter_val_ptr[i] < -1) { + x_grad_ptr[i] = -4 * (2 * y_ptr[i] - 1) * out_grad_ptr[i]; + } else if (inter_val_ptr[i] < 1) { + x_grad_ptr[i] = -2 * (1 - inter_val_ptr[i]) * (2 * y_ptr[i] - 1) * + out_grad_ptr[i]; + } else { + x_grad_ptr[i] = 0; + } + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index b6d320b415e02549e85cb36ab517b0b5433887d5..7047718a3f1bf7e9598952efa1d9bcb20d5cf5b4 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -18,7 +18,6 @@ namespace paddle { namespace operators { using framework::Tensor; -using framework::LoDTensor; class MulOp : public framework::OperatorWithKernel { public: @@ -53,8 +52,9 @@ class MulOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( x_mat_dims[1], y_mat_dims[0], "First matrix's width must be equal with second matrix's height."); - ctx.Output("Out")->Resize( + ctx.Output("Out")->Resize( {x_mat_dims[0], y_mat_dims[1]}); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -83,9 +83,14 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(1) .EqualGreaterThan(1); AddComment(R"DOC( -Two Element Mul Operator. +Mul operator is used to perform matrix multiplication for input X and Y. -The equation is: Out = X * Y +The equation is: + + Out = X * Y + +Both the input `X` and `Y` can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with input `X`. )DOC"); } }; @@ -103,10 +108,8 @@ class MulOpGrad : public framework::OperatorWithKernel { auto x_dims = ctx.Input("X")->dims(); auto y_dims = ctx.Input("Y")->dims(); auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); - auto *x_grad = - ctx.Output(framework::GradVarName("X")); - auto *y_grad = - ctx.Output(framework::GradVarName("Y")); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + auto *y_grad = ctx.Output(framework::GradVarName("Y")); auto x_mat_dims = framework::flatten_to_2d(x_dims, Attr("x_num_col_dims")); diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index a0b1c6b631d97a40d774f7d2ff9550fda9c32db4..375d8a35acc0716259071c31bc332fdf5aabce1c 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -39,8 +39,13 @@ class PadOp : public framework::OperatorWithKernel { for (int i = 0; i < x_dim.size(); ++i) { out_dims[i] = x_dim[i] + paddings[i * 2] + paddings[i * 2 + 1]; } - ctx.Output("Out")->Resize( + ctx.Output("Out")->Resize( framework::make_ddim(out_dims)); + if (out_dims[0] == x_dim[0]) { + // Only pass LoD when the first dimension is equal between + // output and input. + ctx.ShareLoD("X", /*->*/ "Out"); + } } }; @@ -101,7 +106,7 @@ class PadOpGrad : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), "Input(Out@GRAD) should not be null"); auto x_dims = ctx.Input("X")->dims(); - auto *x_g = ctx.Output(framework::GradVarName("X")); + auto *x_g = ctx.Output(framework::GradVarName("X")); if (x_g != nullptr) { x_g->Resize(x_dims); } diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc index 7ae80b296850f2f433c89d904ebf32355b2a29c7..912196c190b5ddbd4e3482a5314e949186b94368 100644 --- a/paddle/operators/prelu_op.cc +++ b/paddle/operators/prelu_op.cc @@ -36,8 +36,9 @@ class PReluOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), "Output(Out) should not be null"); - auto *out = ctx.Output("Out"); + auto *out = ctx.Output("Out"); out->Resize(in->dims()); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -55,6 +56,8 @@ The equation is: f(x) = alpha * x , for x < 0 f(x) = x , for x >= 0 +The input `X` can carry the LoD (Level of Details) information, +or not. And the output shares the LoD with input `X`. )DOC"); } }; @@ -69,11 +72,11 @@ class PReluGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), "Input(Out@GRAD) should not be null"); - auto *dx = ctx.Output(framework::GradVarName("X")); + auto *dx = ctx.Output(framework::GradVarName("X")); auto *x = ctx.Input("X"); auto *dalpha = - ctx.Output(framework::GradVarName("Alpha")); + ctx.Output(framework::GradVarName("Alpha")); auto *alpha = ctx.Input("Alpha"); dx->Resize(x->dims()); diff --git a/paddle/operators/prelu_op.h b/paddle/operators/prelu_op.h index 3269116c112f115e1e8fbbee0dc3b81dbe736e69..6b78ed295cbac060d816fb3dd27a4b80145cb1ce 100644 --- a/paddle/operators/prelu_op.h +++ b/paddle/operators/prelu_op.h @@ -96,7 +96,7 @@ class PReluGradKernel : public framework::OpKernel { trans(context.device_context(), out_ptr, out_ptr + numel, dout_ptr, dx_ptr, PReluGradFunctor(alpha_ptr)); - // TODO (Zhuoyuan): add dalpha upgrade when GPU kernels ready + // TODO(Zhuoyuan): add dalpha upgrade when GPU kernels ready } }; diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..39af08c8751c3b95cf5fdef7395186a0176a20a2 --- /dev/null +++ b/paddle/operators/rank_loss_op.cc @@ -0,0 +1,126 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/rank_loss_op.h" + +namespace paddle { +namespace operators { + +class RankLossOp : public framework::OperatorWithKernel { + public: + RankLossOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + // input check + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) shouldn't be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), + "Input(Left) shouldn't be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), + "Input(Right) shouldn't be null"); + auto label_dims = ctx.Input("Label")->dims(); + auto left_dims = ctx.Input("Left")->dims(); + auto right_dims = ctx.Input("Right")->dims(); + PADDLE_ENFORCE((label_dims == left_dims) && (left_dims == right_dims), + "All inputs must have the same size"); + PADDLE_ENFORCE((label_dims.size() == 2) && (label_dims[1] == 1), + "All inputs must be row vector with size batch_size x 1."); + ctx.Output("Out")->Resize(label_dims); + } +}; + +class RankLossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + RankLossOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Label", + "The label indicating A ranked higher than B or not, row vector."); + AddInput("Left", "The output of RankNet for doc A, vector."); + AddInput("Right", "The output of RankNet for doc B, vetor"); + AddOutput("Out", "The output loss of RankLoss operator, vector."); + AddComment(R"DOC(RankLoss operator + +Rank loss operator for RankNet[1]. RankNet is a pairwise ranking model with +one training sample consisting of a pair of doc A and B, and the label P +indicating that A is ranked higher than B or not: + +P = {0, 1} or {0, 0.5, 1}, where 0.5 means no information about the rank of +the input pair. + +The RankLoss operator contains three inputs: Left (o_i), Right (o_j) and Label +(P_{i,j}), which represent the output of RankNet for two docs and the label +respectively, and yields the rank loss C_{i,j} by following the expression + +\f[ + C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + log(1 + e^{o_{i,j}}) \\ + o_{i,j} = o_i - o_j \\ + \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} +\f] + +The operator can take inputs of one sample or in batch. + +[1]. Chris Burges, Tal Shaked, Erin Renshaw, et al. Learning to + Rank using Gradient Descent. + http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf +)DOC"); + } +}; + +class RankLossGradOp : public framework::OperatorWithKernel { + public: + RankLossGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), + "Input(Left) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), + "Input(Right) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto dims = ctx.Input("Left")->dims(); + auto *left_grad = + ctx.Output(framework::GradVarName("Left")); + auto *right_grad = + ctx.Output(framework::GradVarName("Right")); + if (left_grad) { + left_grad->Resize(dims); + } + if (right_grad) { + right_grad->Resize(dims); + } + } +}; + +} // namespace operators +} // namespace paddle +namespace ops = paddle::operators; + +REGISTER_OP(rank_loss, ops::RankLossOp, ops::RankLossOpMaker, rank_loss_grad, + ops::RankLossGradOp); +REGISTER_OP_CPU_KERNEL(rank_loss, + ops::RankLossKernel); +REGISTER_OP_CPU_KERNEL( + rank_loss_grad, ops::RankLossGradKernel); diff --git a/paddle/operators/rank_loss_op.cu b/paddle/operators/rank_loss_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..779588ff36c792b8925a535d60f1cfbbe3c66d86 --- /dev/null +++ b/paddle/operators/rank_loss_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/rank_loss_op.h" + +REGISTER_OP_GPU_KERNEL( + rank_loss, + paddle::operators::RankLossKernel); +REGISTER_OP_GPU_KERNEL( + rank_loss_grad, + paddle::operators::RankLossGradKernel); diff --git a/paddle/operators/rank_loss_op.h b/paddle/operators/rank_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..7df195ff47ecfd79388385eed4bd37b8c9b45979 --- /dev/null +++ b/paddle/operators/rank_loss_op.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class RankLossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* out_t = ctx.Output("Out"); + auto* label_t = ctx.Input("Label"); + auto* left_t = ctx.Input("Left"); + auto* right_t = ctx.Input("Right"); + out_t->mutable_data(ctx.GetPlace()); + + auto out = framework::EigenVector::Flatten(*out_t); + auto label = framework::EigenVector::Flatten(*label_t); + auto left = framework::EigenVector::Flatten(*left_t); + auto right = framework::EigenVector::Flatten(*right_t); + + auto& dev = ctx.GetEigenDevice(); + out.device(dev) = + (1. + (left - right).exp()).log() - label * (left - right); + } +}; + +template +class RankLossGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* d_left_t = + ctx.Output(framework::GradVarName("Left")); + auto* d_right_t = + ctx.Output(framework::GradVarName("Right")); + + auto* d_out_t = ctx.Input(framework::GradVarName("Out")); + auto* label_t = ctx.Input("Label"); + auto* left_t = ctx.Input("Left"); + auto* right_t = ctx.Input("Right"); + + auto& dev = ctx.GetEigenDevice(); + auto d_out = framework::EigenVector::Flatten(*d_out_t); + auto label = framework::EigenVector::Flatten(*label_t); + auto left = framework::EigenVector::Flatten(*left_t); + auto right = framework::EigenVector::Flatten(*right_t); + + // compute d_left + if (d_left_t) { + d_left_t->mutable_data(ctx.GetPlace()); + auto d_left = framework::EigenVector::Flatten(*d_left_t); + d_left.device(dev) = d_out * (1. / (1. + (right - left).exp()) - label); + } + // compute d_right + if (d_right_t) { + d_right_t->mutable_data(ctx.GetPlace()); + auto d_right = framework::EigenVector::Flatten(*d_right_t); + d_right.device(dev) = + -d_out * (1.0 / (1. + (right - left).exp()) - label); + } + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index d3413d7cb9305732e9ddf3cb1bc267f7203097f3..ad985839f5908d9235a4dbefc9b841362810114e 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -29,9 +29,11 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; void RecurrentAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = scope.FindVar((arg_->inlinks[0]).external) - ->GetMutable() - ->dims()[0]; + auto* input0 = scope.FindVar(arg_->inlinks[0]); + PADDLE_ENFORCE_NOT_NULL(input0); + seq_len_ = input0->GetMutable()->dims()[0]; + PADDLE_ENFORCE_GT(seq_len_, 0); + CreateScopes(scope); auto step_scopes = GetStepScopes(scope); rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, @@ -123,14 +125,12 @@ void RecurrentAlgorithm::InitMemories(Scope* step_scope, } const rnn::ArgumentName RecurrentOp::kArgName{ - "step_net", "step_scopes", "inlinks", - "outlinks", "inlink_alias", "outlink_alias", + "step_net", "step_scopes", "inlinks", "outlinks", "memories", "pre_memories", "boot_memories"}; const rnn::ArgumentName RecurrentGradientOp::kArgName{ - "step_net", "step_scopes", "outlink@grad", - "inlink@grad", "inlink_alias", "outlink_alias", - "memories", "pre_memories", "boot_memories@grad"}; + "step_net", "step_scopes", "outlink@grad", "inlink@grad", + "memories", "pre_memories", "boot_memories@grad"}; RecurrentOp::RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs, @@ -160,8 +160,6 @@ class RecurrentAlgorithmProtoAndCheckerMaker AddOutput(name.step_scopes, "step scopes"); // Attributes stored in AttributeMap - AddAttr>(name.inlink_alias, "alias of inlinks"); - AddAttr>(name.outlink_alias, "alias of outlinks"); AddAttr>(name.pre_memories, "names of pre-memories"); AddAttr>(name.memories, "names of memories"); @@ -206,9 +204,8 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients( } void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = scope.FindVar((arg_->inlinks[0]).external) - ->GetMutable() - ->dims()[0]; + seq_len_ = + scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; auto step_scopes = GetStepScopes(scope); rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, true /*infer_shape_mode*/); diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 0d05e344148c68f5625dd819ec59c5991892e4ce..ddb93007e21e4d1ae4be3650019c8bc6a680252d 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -50,7 +50,12 @@ class ReshapeOp : public framework::OperatorWithKernel { std::transform(shape.begin(), shape.end(), shape_int64.begin(), [](int a) { return static_cast(a); }); auto out_dims = framework::make_ddim(shape_int64); - ctx.Output("Out")->Resize(out_dims); + ctx.Output("Out")->Resize(out_dims); + if (shape[0] == in->dims()[0]) { + // Only pass LoD when the first dimension is equal between + // output and input. + ctx.ShareLoD("X", /*->*/ "Out"); + } } }; @@ -94,7 +99,7 @@ class ReshapeGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), "Input(Out@GRAD) shouldn't be null."); auto dims = ctx.Input("X")->dims(); - auto *d_in = ctx.Output(framework::GradVarName("X")); + auto *d_in = ctx.Output(framework::GradVarName("X")); d_in->Resize(dims); } }; diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index 6c082cb1825e04accb09019fef28eb2ec6523a5b..ca7219b26d83eb6b8db75a5ed9cd360c5ac1d5df 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -24,22 +24,23 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; void SegmentInputs(const std::vector& step_scopes, - const std::vector& inlinks, const size_t seq_len, - bool infer_shape_mode) { + const std::vector& inlinks, + const size_t seq_len, bool infer_shape_mode) { PADDLE_ENFORCE(!inlinks.empty(), "no in links are provided."); for (size_t i = 0; i < inlinks.size(); ++i) { - auto input_var = step_scopes[0]->FindVar(inlinks[i].external); - PADDLE_ENFORCE(input_var != nullptr, "input link [%s] is not in scope.", - inlinks[i].external); + // global inputs + auto input_var = step_scopes[0]->parent().FindVar(inlinks[i]); + PADDLE_ENFORCE_NOT_NULL(input_var, "input link [%s] is not in scope.", + inlinks[i]); LoDTensor* input = input_var->GetMutable(); f::DDim dims = input->dims(); - PADDLE_ENFORCE(static_cast(dims[0]) == seq_len, - "all the inlinks must have same length"); + PADDLE_ENFORCE_EQ(static_cast(dims[0]), seq_len, + "all the inlinks be the same length"); f::DDim step_dims = slice_ddim(dims, 1, dims.size()); for (size_t j = 0; j < seq_len; j++) { Tensor* step_input = - step_scopes[j]->NewVar(inlinks[i].internal)->GetMutable(); + step_scopes[j]->NewVar(inlinks[i])->GetMutable(); if (!infer_shape_mode) { // The input of operators of each step is Tensor here. // Maybe need to modify Slice function. @@ -51,18 +52,17 @@ void SegmentInputs(const std::vector& step_scopes, } void ConcatOutputs(const std::vector& step_scopes, - const std::vector& outlinks, const size_t seq_len, - bool infer_shape_mode) { + const std::vector& outlinks, + const size_t seq_len, bool infer_shape_mode) { for (size_t i = 0; i < outlinks.size(); i++) { - auto output_var = step_scopes[0]->FindVar(outlinks[i].external); - PADDLE_ENFORCE(output_var != nullptr, "output link [%s] is not in scope.", - outlinks[i].external); + auto output_var = step_scopes[0]->parent().FindVar(outlinks[i]); + PADDLE_ENFORCE_NOT_NULL(output_var, "output link [%s] is not in scope.", + outlinks[i]); LoDTensor* output = output_var->GetMutable(); if (infer_shape_mode) { - auto step_scope_var = step_scopes[0]->FindVar(outlinks[i].internal); - PADDLE_ENFORCE(step_scope_var != nullptr, "%s not in scope", - outlinks[i].internal); + auto step_scope_var = step_scopes[0]->FindVar(outlinks[i]); + PADDLE_ENFORCE_NOT_NULL(step_scope_var, "%s not in scope", outlinks[i]); f::DDim step_dims = step_scope_var->template GetMutable()->dims(); std::vector dims_vec = vectorize(step_dims); @@ -71,9 +71,8 @@ void ConcatOutputs(const std::vector& step_scopes, } else { output->mutable_data(platform::CPUPlace()); for (size_t j = 0; j < seq_len; j++) { - LoDTensor* step_output = step_scopes[j] - ->FindVar(outlinks[i].internal) - ->GetMutable(); + LoDTensor* step_output = + step_scopes[j]->FindVar(outlinks[i])->GetMutable(); // TODO(luotao02) data type and platform::DeviceContext() should set // correctly (output->Slice(j, j + 1)) @@ -113,29 +112,9 @@ void InitArgument(const ArgumentName& name, Argument* arg, const framework::OperatorBase& op) { arg->step_scopes = op.Output(name.step_scopes); - auto inlinks = op.Inputs(name.inlinks); - auto inlink_alias = op.Attr>(name.inlink_alias); - PADDLE_ENFORCE(inlinks.size() == inlink_alias.size(), - "the size of inlinks and inlink_alias don't match:%d,%d", - inlinks.size(), inlink_alias.size()); - for (size_t i = 0; i < inlinks.size(); ++i) { - rnn::Link link; - link.external = inlinks[i]; - link.internal = inlink_alias[i]; - (arg->inlinks).push_back(link); - } + arg->inlinks = op.Inputs(name.inlinks); - auto outlinks = op.Outputs(name.outlinks); - auto outlink_alias = op.Attr>(name.outlink_alias); - PADDLE_ENFORCE(outlinks.size() == outlink_alias.size(), - "the size of outlinks and outlink_alias don't match:%d,%d", - outlinks.size(), outlink_alias.size()); - for (size_t i = 0; i < outlinks.size(); ++i) { - rnn::Link link; - link.external = outlinks[i]; - link.internal = outlink_alias[i]; - (arg->outlinks).push_back(link); - } + arg->outlinks = op.Outputs(name.outlinks); auto boot_memories = op.Inputs(name.boot_memories); diff --git a/paddle/operators/rnn/recurrent_op_utils.h b/paddle/operators/rnn/recurrent_op_utils.h index 17941c503cfcc83415b8bc635623a2c2ce2981c3..7dafe5d0088c4c8bf2cad163654e7e4f28eebe2e 100644 --- a/paddle/operators/rnn/recurrent_op_utils.h +++ b/paddle/operators/rnn/recurrent_op_utils.h @@ -41,18 +41,11 @@ struct MemoryAttr { std::string boot_var; }; -struct Link { - // input or output links name. - std::string internal; - // alias to avoid duplicate keys in scopes. - std::string external; -}; - struct Argument { std::string step_net; std::string step_scopes; - std::vector inlinks; - std::vector outlinks; + std::vector inlinks; + std::vector outlinks; std::vector memories; }; @@ -61,8 +54,6 @@ struct ArgumentName { std::string step_scopes; std::string inlinks; std::string outlinks; - std::string inlink_alias; // the alias of inlinks in step net. - std::string outlink_alias; // the alias of outlinks in step net. std::string memories; // the memory name std::string pre_memories; // the previous memory name std::string boot_memories; // the boot memory name @@ -72,15 +63,15 @@ struct ArgumentName { * Prepare inputs for each step net. */ void SegmentInputs(const std::vector& step_scopes, - const std::vector& inlinks, const size_t seq_len, - bool infer_shape_mode); + const std::vector& inlinks, + const size_t seq_len, bool infer_shape_mode); /** * Process outputs of step nets and merge to variables. */ void ConcatOutputs(const std::vector& step_scopes, - const std::vector& outlinks, const size_t seq_len, - bool infer_shape_mode); + const std::vector& outlinks, + const size_t seq_len, bool infer_shape_mode); void LinkMemories(const std::vector& step_scopes, const std::vector& memories, const size_t step_id, diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 2a3fd3be941d91aaa6b014df91d3025f07767577..fc3ad721f210213491617452141dfa8834b067c0 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -44,7 +44,8 @@ class RowwiseAddOp : public framework::OperatorWithKernel { framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims, "The width of two operands must be same"); PADDLE_ENFORCE_EQ(ctx.OutputSize("Out"), 1, "The output size must be 1"); - ctx.Output("Out")->Resize(x_dims); + ctx.Output("Out")->Resize(x_dims); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -83,8 +84,8 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ( framework::slice_ddim(x_dims, num_col_dims, x_dims.size()), b_dims, "The width of two operands must be same"); - auto *dx = ctx.Output(framework::GradVarName("X")); - auto *db = ctx.Output(framework::GradVarName("b")); + auto *dx = ctx.Output(framework::GradVarName("X")); + auto *db = ctx.Output(framework::GradVarName("b")); if (dx) dx->Resize(x_dims); if (db) db->Resize(b_dims); } diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index d1f42e8662537d35e17429f9d436fdc0e5a1dc11..1ae77a9722ef1a5548a6c4100c32fdddcee8c5cd 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -33,8 +33,9 @@ class ScaleOp : public framework::OperatorWithKernel { "Output(Out) of ScaleOp should not be null."); auto *in = ctx.Input("X"); - auto *out = ctx.Output("Out"); + auto *out = ctx.Output("Out"); out->Resize(in->dims()); + ctx.ShareLoD("X", /*->*/ "Out"); } }; diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc index 8820262732327306f4f807702751708bd1e2aa36..3f02081a060281dec533c02b346f0667da28b8c3 100644 --- a/paddle/operators/scatter_op.cc +++ b/paddle/operators/scatter_op.cc @@ -44,7 +44,7 @@ class ScatterOp : public framework::OperatorWithKernel { framework::DDim data_dim(ctx.Input("Updates")->dims()); for (int i = 1; i < data_dim.size(); ++i) PADDLE_ENFORCE_EQ(data_dim[i], ctx.Input("Updates")->dims()[i]); - ctx.Output("Out")->Resize( + ctx.Output("Out")->Resize( ctx.Input("Ref")->dims()); } }; @@ -56,10 +56,9 @@ class ScatterGradOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { auto *dUpdates = - ctx.Output(framework::GradVarName("Updates")); + ctx.Output(framework::GradVarName("Updates")); auto *Updates = ctx.Input("Updates"); - auto *dRef = - ctx.Output(framework::GradVarName("Ref")); + auto *dRef = ctx.Output(framework::GradVarName("Ref")); auto *Ref = ctx.Input("Ref"); dRef->Resize(Ref->dims()); diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 1232e64c7f0132b9ea19b3d7e1ebe9531e1e25a5..b063e2427217f20eb89f7cd1af0354ad0e400feb 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -33,7 +33,7 @@ class SGDOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ctx.Input("param")->dims(), ctx.Input("grad")->dims(), "Two input of SGD Op's dimension must be same."); - ctx.Output("param_out") + ctx.Output("param_out") ->Resize(ctx.Input("param")->dims()); } }; diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc deleted file mode 100644 index 992b19965e0ca9ce7dba1b8b3c5b7780af06eb45..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.cc +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/sigmoid_op.h" - -namespace paddle { -namespace operators { - -class SigmoidOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of SigmoidOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) of SigmoidOp should not be null."); - - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); - } -}; - -class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { - public: - SigmoidOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "sigmoid input"); - AddOutput("Y", "sigmoid output"); - AddComment("Sigmoid function"); - } -}; - -class SigmoidOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("Y")->dims()); - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad, - ops::SigmoidOpGrad); -REGISTER_OP_CPU_KERNEL(sigmoid, - ops::SigmoidKernel); -REGISTER_OP_CPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); diff --git a/paddle/operators/sigmoid_op.h b/paddle/operators/sigmoid_op.h deleted file mode 100644 index b01a9b3f23283471f8846325075719ba0e75ed35..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; - -template -class SigmoidKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto input = context.Input("X"); - auto output = context.Output("Y"); - output->mutable_data(context.GetPlace()); - - // The clipping is used in Paddle's raw implenmention - auto X = EigenVector::Flatten(*input); - auto Y = EigenVector::Flatten(*output); - auto place = context.GetEigenDevice(); - - Y.device(place) = 1. / (1. + (-X).exp()); - } -}; - -template -class SigmoidGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto Y_t = context.Input("Y"); - auto dY_t = context.Input(framework::GradVarName("Y")); - auto dX_t = context.Output(framework::GradVarName("X")); - - dX_t->mutable_data(context.GetPlace()); - - auto dX = EigenVector::Flatten(*dX_t); - auto Y = EigenVector::Flatten(*Y_t); - auto dY = EigenVector::Flatten(*dY_t); - dX.device(context.GetEigenDevice()) = dY * Y * (1. - Y); - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..ae6d1c80b300690b070024d6266a1b99bf2ef04f --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -0,0 +1,133 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/smooth_l1_loss_op.h" + +namespace paddle { +namespace operators { + +class SmoothL1LossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Y must be initialized."); + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "The shape of X and Y must be the same."); + PADDLE_ENFORCE_GE(x->dims().size(), 2, + "The tensor rank of X must be at least 2."); + auto* inside_weight = ctx.Input("InsideWeight"); + if (inside_weight) { + auto* outside_weight = ctx.Input("OutsideWeight"); + PADDLE_ENFORCE_NOT_NULL(outside_weight, + "If weights are provided, must specify both " + "inside and outside weights."); + PADDLE_ENFORCE_EQ(inside_weight->dims(), x->dims(), + "The shape of InsideWeight must be same as X."); + PADDLE_ENFORCE_EQ(outside_weight->dims(), x->dims(), + "The shape of OutsideWeight must be same as X."); + } + + auto* diff = ctx.Output("Diff"); + auto* out = ctx.Output("Out"); + diff->Resize(x->dims()); + // loss is a two-rank tensor + out->Resize({x->dims()[0], 1}); + } +}; + +template +class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SmoothL1LossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input tensor of smooth l1 loss op." + "The rank should be greater or equal to 2 with shape " + "[batch_size, value_dim1, value_dim2, ..., value_dimN]"); + AddInput("Y", + "The target tensor of smooth l1 loss op " + "with the same shape as X."); + AddInput("InsideWeight", + "Optional input tensor of smooth l1 loss op with the same shape " + "as X. If provided, the result of (X - Y) will be multiplied " + "by this tensor element by element."); + AddInput("OutsideWeight", + "Optinal input of smooth l1 loss op with the same shape as X." + "If provided, the output smooth l1 loss will be multiplied by " + "this tensor element by element."); + AddOutput("Diff", "Intermediate variable to cache InsideWeight*(X-Y).") + .AsIntermediate(); + AddOutput("Out", "Smooth l1 loss."); + AddAttr("sigma", + "Hyper parameter of smooth l1 loss op." + "A float scalar with default value 3.0.") + .SetDefault(3.0); + AddComment(R"DOC( +Compute smooth l1 loss for input and target. The operator take the 1st +dimension of input as batch size. For each instance, it will compute +smooth l1 loss element by element first and sum all losses to one value. +So the output shape is [batch_size, 1]. + +The equation is: +loss = 0.5 * (sigma * (x-y))^2 if abs(x - y) < 1 / sigma^2 + abs(x - y) - 0.5 / sigma^2 otherwise + +)DOC"); + } +}; + +class SmoothL1LossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + auto in_dims = ctx.Input("X")->dims(); + auto out_dims = + ctx.Input(framework::GradVarName("Out"))->dims(); + auto* x_grad = ctx.Output(framework::GradVarName("X")); + auto* y_grad = ctx.Output(framework::GradVarName("Y")); + + PADDLE_ENFORCE_GE(out_dims.size(), 2, + "The tensor rank of Input(Out@Grad) should be 2."); + PADDLE_ENFORCE_EQ(out_dims[0], in_dims[0], + "The 1st dimension of Input(Out@Grad) must be " + "same as input."); + PADDLE_ENFORCE_EQ(out_dims[1], 1, + "The 2nd dimension of Input(Out@Grad) must be 1."); + + if (x_grad) x_grad->Resize(in_dims); + if (y_grad) y_grad->Resize(in_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(smooth_l1_loss, ops::SmoothL1LossOp, + ops::SmoothL1LossOpMaker, smooth_l1_loss_grad, + ops::SmoothL1LossGradOp); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/sigmoid_op.cu b/paddle/operators/smooth_l1_loss_op.cu similarity index 73% rename from paddle/operators/sigmoid_op.cu rename to paddle/operators/smooth_l1_loss_op.cu index 1a50dfe14a7b9e2614aadb7729de9f9e461e9905..1c3172f43867741cd1f26979a366b2425f326321 100644 --- a/paddle/operators/sigmoid_op.cu +++ b/paddle/operators/smooth_l1_loss_op.cu @@ -13,11 +13,12 @@ limitations under the License. */ #define EIGEN_USE_GPU -#include "paddle/operators/sigmoid_op.h" -namespace ops = paddle::operators; +#include "paddle/operators/smooth_l1_loss_op.h" -REGISTER_OP_GPU_KERNEL(sigmoid, - ops::SigmoidKernel); +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); REGISTER_OP_GPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..0604fb5e1c2f17c702208520a1d23bd5c3c65b5d --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.h @@ -0,0 +1,182 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; +template +using EigenMatrix = framework::EigenMatrix; + +template +struct SmoothL1LossForward { + HOSTDEVICE SmoothL1LossForward(const T& sigma2) : sigma2(sigma2) {} + + HOSTDEVICE T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return 0.5 * val * val * sigma2; + } else { + return abs_val - 0.5 / sigma2; + } + } + + T sigma2; +}; + +template +class SmoothL1LossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* in2 = context.Input("InsideWeight"); + auto* in3 = context.Input("OutsideWeight"); + auto* out0 = context.Output("Diff"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto sigma = static_cast(context.Attr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in2 != nullptr) && (in3 != nullptr); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + auto diff = EigenVector::Flatten(*out0); + + diff.device(place) = x - y; + // multiply inside weight + if (has_weight) { + auto inside_weight = EigenVector::Flatten(*in2); + // cache diff, reused in bp + diff.device(place) = diff * inside_weight; + } + + auto in_counts = in0->numel(); + Tensor ptensor_errors; + ptensor_errors.mutable_data({static_cast(in_counts)}, + context.GetPlace()); + auto errors = EigenVector::Flatten(ptensor_errors); + // apply smooth l1 forward + errors.device(place) = diff.unaryExpr(SmoothL1LossForward(sigma2)); + + // multiply outside weight + if (has_weight) { + auto outside_weight = EigenVector::Flatten(*in3); + errors.device(place) = errors * outside_weight; + } + auto loss = EigenVector::Flatten(*out1); + // first dimension of 'X' is the number of samples + auto mat_dims = + framework::make_ddim({static_cast(in0->dims()[0]), + static_cast(in_counts / in0->dims()[0])}); + auto errors_mat_view = EigenMatrix::From(ptensor_errors, mat_dims); + loss.device(place) = errors_mat_view.sum(Eigen::array({{1}})); + } +}; + +template +struct SmoothL1LossBackward { + HOSTDEVICE SmoothL1LossBackward(const T& sigma2) : sigma2(sigma2) {} + + HOSTDEVICE T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return sigma2 * val; + } else { + return (0 < val) - (val < 0); + } + } + + T sigma2; +}; + +template +class SmoothL1LossGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("InsideWeight"); + auto* in1 = context.Input("OutsideWeight"); + auto* in2 = context.Input("Diff"); + auto* og = context.Input(framework::GradVarName("Out")); + auto sigma = static_cast(context.Attr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in0 != nullptr) && (in1 != nullptr); + + auto place = context.GetEigenDevice(); + + auto in_dims = in2->dims(); + auto counts = in2->numel(); + auto cols = counts / in_dims[0]; + auto mat_dims = framework::make_ddim( + {static_cast(in_dims[0]), static_cast(cols)}); + + Tensor ptensor_diff; + ptensor_diff.mutable_data({static_cast(counts)}, + context.GetPlace()); + auto diff = EigenVector::Flatten(ptensor_diff); + // apply smooth l1 backwoard + diff.device(place) = EigenVector::Flatten(*in2).unaryExpr( + SmoothL1LossBackward(sigma2)); + + // compute weights + Tensor ptensor_weights; + ptensor_weights.mutable_data(mat_dims, context.GetPlace()); + auto weights = EigenMatrix::From(ptensor_weights); + // initialize to 1.0 + weights.device(place) = weights.constant(static_cast(1.0)); + if (has_weight) { + auto inside_weight = EigenMatrix::From(*in0, mat_dims); + auto outside_weight = EigenMatrix::From(*in1, mat_dims); + weights.device(place) = inside_weight * outside_weight; + } + + // compute gradients + auto out_grad = EigenMatrix::From(*og); + auto diff_mat_view = EigenMatrix::From(ptensor_diff, mat_dims); + auto gradients = out_grad.broadcast( + Eigen::array({{1, static_cast(cols)}})) * + weights * diff_mat_view; + + auto* out0 = context.Output(framework::GradVarName("X")); + auto* out1 = context.Output(framework::GradVarName("Y")); + + if (out0) { + out0->mutable_data(context.GetPlace()); + auto x_grad = EigenMatrix::From(*out0, mat_dims); + x_grad.device(place) = gradients; + } + + if (out1) { + out1->mutable_data(context.GetPlace()); + auto y_grad = EigenMatrix::From(*out1, mat_dims); + y_grad.device(place) = -1 * gradients; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index c67eb028c882ed82ca4e6a4dd70cdea9f69cdc24..e15cfe485016552971924a40a172e74a90629dce 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -30,8 +30,7 @@ class SoftmaxOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx.Input("X")->dims().size() == 2UL, "The input of softmax op must be a matrix."); - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); + ctx.Output("Y")->Resize(ctx.Input("X")->dims()); } }; @@ -77,7 +76,7 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel { ctx.Input(framework::GradVarName("Y"))->dims(), "Input(Y) and its gradients should have a same shape."); - ctx.Output(framework::GradVarName("X")) + ctx.Output(framework::GradVarName("X")) ->Resize(ctx.Input("X")->dims()); } }; diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc index 61296f5c8122fdce7083e9a91dc313482875c805..a9d35b4fb79ae83379552ae2c2b4d694bd8f86dd 100644 --- a/paddle/operators/split_op.cc +++ b/paddle/operators/split_op.cc @@ -27,7 +27,7 @@ class SplitOp : public framework::OperatorWithKernel { void InferShape(const framework::InferShapeContext &ctx) const override { // infershape auto *in = ctx.Input("X"); - auto outs = ctx.MultiOutput("Out"); + auto outs = ctx.MultiOutput("Out"); size_t axis = static_cast(ctx.Attr("axis")); size_t num = static_cast(ctx.Attr("num")); std::vector sections = diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc index 39f4305877de20d451bc35fe698a0eabf9758d57..33a564b05b1b490c6d23b7d17cef45b7740dfa39 100644 --- a/paddle/operators/squared_l2_distance_op.cc +++ b/paddle/operators/squared_l2_distance_op.cc @@ -54,9 +54,10 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel { "First dimension of target must be equal to input " "or to 1."); - ctx.Output("sub_result") + ctx.Output("sub_result") ->Resize({x_dims[0], x->numel() / x_dims[0]}); - ctx.Output("Out")->Resize({x_dims[0], 1}); + ctx.Output("Out")->Resize({x_dims[0], 1}); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -79,6 +80,9 @@ class SquaredL2DistanceOpMaker : public framework::OpProtoAndCheckerMaker { input or to 1. If the first dimension of target is 1, SquaredL2DistanceOp will broadcast target's first dimension to input's first dimension. You can decide whether calculate the gradient of input and target. + + Both the input X and Y can carry the LoD (Level of Details) information, + or not. But the output only shares the LoD with input X. )DOC"); } }; @@ -100,10 +104,8 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(out_dims[1], 1, "Second dimension of output gradient " "must be 1."); - auto* x_grad = - ctx.Output(framework::GradVarName("X")); - auto* y_grad = - ctx.Output(framework::GradVarName("Y")); + auto* x_grad = ctx.Output(framework::GradVarName("X")); + auto* y_grad = ctx.Output(framework::GradVarName("Y")); if (x_grad) x_grad->Resize(x_dims); if (y_grad) y_grad->Resize(y_dims); } diff --git a/paddle/operators/strided_memcpy.h b/paddle/operators/strided_memcpy.h new file mode 100644 index 0000000000000000000000000000000000000000..c9dd80518424017d9834a2bf7aee14caa56c9d79 --- /dev/null +++ b/paddle/operators/strided_memcpy.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/operators/detail/strided_memcpy.h" + +namespace paddle { +namespace operators { + +// Strided memory copy from src to dst. +// +// The src and dst should be both on dev_ctx.GetPlace(), otherwise, there will +// be a segment fault. +// +// The stride of an array (also referred to as increment, pitch or step size) is +// the number of locations in memory between beginnings of successive array +// elements +// +// For example, for tensor like [1, 3, 300, 300]. If there is no padding, the +// stride is [270000, 90000, 300, 1]. +// +// NOTE: When use GPU, the memcpy is async. To sync memcpy, please invoke +// `dev_ctx.Wait()`. +template +inline void StridedMemcpy(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_dim, + const framework::DDim& dst_stride, T* dst) { + using namespace detail; + StridedCopyDimVisitor func(dev_ctx, src, src_stride, dst_stride, dst); + boost::apply_visitor(func, dst_dim); +} +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..05882a88738cfc9cc23480efe0afe504008377ca --- /dev/null +++ b/paddle/operators/strided_memcpy_test.cc @@ -0,0 +1,160 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/strided_memcpy.h" +#include "gtest/gtest.h" +#include "paddle/memory/memory.h" + +namespace paddle { +namespace operators { + +TEST(StridedMemcpy, CPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + framework::DDim src_stride({5, 1}); + + int dst[4]; + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CPUDeviceContext ctx; + StridedMemcpy(ctx, src + 1, src_stride, dst_dim, dst_stride, dst); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); +} + +TEST(StridedMemcpy, CPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + int dst[8]; + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CPUDeviceContext ctx; + + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst); + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst + 2); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } +} + +#ifndef PADDLE_ONLY_CPU +TEST(StridedMemcpy, GPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + framework::DDim src_stride({5, 1}); + + int dst[4]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CUDADeviceContext ctx(gpu0); + StridedMemcpy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, + gpu_dst); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +TEST(StridedMemcpy, GPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + int dst[8]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CUDADeviceContext ctx(gpu0); + + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst); + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, + gpu_dst + 2); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +#endif +} // namespace operators +} // namespace paddle \ No newline at end of file diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 41e05c27f9029b2664685d3979fadcfd2bf6dbce..437fc262f359525045a4d772ee2c204ef571caa7 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -28,7 +28,7 @@ class SumOp : public framework::OperatorWithKernel { "Output(Out) of SumOp should not be null."); auto ins = ctx.MultiInput("X"); - auto *out = ctx.Output("Out"); + auto *out = ctx.Output("Out"); int N = ins.size(); auto in_dim = ins[0]->dims(); @@ -39,6 +39,7 @@ class SumOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(in_dim == dim, "Input tensors must have same shape"); } out->Resize(in_dim); + ctx.ShareLoD("X", /*->*/ "Out"); } }; @@ -49,8 +50,11 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "the input tensors of sum operator.").AsDuplicable(); AddOutput("Out", "the output tensor of sum operator."); AddComment(R"DOC( - Sum the input tensors. - )DOC"); +Sum the input tensors. + +All the inputs can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD with the first input. +)DOC"); } }; @@ -61,7 +65,7 @@ class SumGradOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { auto outputs = - ctx.MultiOutput(framework::GradVarName("X")); + ctx.MultiOutput(framework::GradVarName("X")); auto dims = ctx.Input(framework::GradVarName("Out"))->dims(); for (auto output : outputs) { output->Resize(dims); diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc index 169b815feffd86f9ff04c129ccc997230ce03a8c..a6e43964e9825cd1ced9e7c1bc8d691422248fee 100644 --- a/paddle/operators/top_k_op.cc +++ b/paddle/operators/top_k_op.cc @@ -40,8 +40,8 @@ class TopkOp : public framework::OperatorWithKernel { framework::DDim dims = input->dims(); dims[dims.size() - 1] = k; - ctx.Output("Out")->Resize(dims); - ctx.Output("Indices")->Resize(dims); + ctx.Output("Out")->Resize(dims); + ctx.Output("Indices")->Resize(dims); } }; diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..017a05326e9b397185d7c3530891884b11784783 --- /dev/null +++ b/paddle/operators/transpose_op.cc @@ -0,0 +1,118 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/transpose_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class TransposeOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + std::vector axis = ctx.Attr>("axis"); + size_t x_rank = x_dims.size(); + size_t axis_size = axis.size(); + + PADDLE_ENFORCE_EQ(x_rank, axis_size, + "the input tensor's rank(%d) " + "should be equal to the axis's size(%d)", + x_rank, axis_size); + + std::vector count(axis_size, 0); + for (size_t i = 0; i < axis_size; i++) { + PADDLE_ENFORCE( + axis[i] < static_cast(axis_size) && ++count[axis[i]] == 1, + "Each element of Attribute axis should be a unique value " + "range from 0 to (dims - 1), " + "where the dims is the axis's size"); + } + + framework::DDim out_dims(x_dims); + for (size_t i = 0; i < axis_size; i++) { + out_dims[i] = x_dims[axis[i]]; + } + ctx.Output("Out")->Resize(out_dims); + } +}; + +class TransposeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TransposeOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "X", + "(Tensor)The input tensor, tensors with rank at most 6 are supported"); + AddOutput("Out", "(Tensor)The output tensor"); + AddAttr>( + "axis", + "(vector)a list of values, and the size of the list should be " + "the same with the input tensor rank, the tensor will " + "permute the axes according the the values given"); + AddComment(R"DOC( +The Tensor will be permuted according to the axis values given. +The op is very much like the numpy.transpose function in python +For example: + >> input = numpy.arange(6).reshape((2,3)) + >> input + array([[0, 1, 2], + [3, 4, 5]]) + >> axis = [1, 0] + >> output = input.transpose(axis) + >> output + array([[0, 3], + [1, 4], + [2, 5]]) +So, given a input tensor of shape(N, C, H, W) and the axis is {0, 2, 3, 1}, +the output tensor shape will be (N, H, W, C) +)DOC"); + } +}; + +class TransposeOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + + if (x_grad) x_grad->Resize(x_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(transpose, ops::TransposeOp, ops::TransposeOpMaker, transpose_grad, + ops::TransposeOpGrad); +REGISTER_OP_CPU_KERNEL(transpose, + ops::TransposeKernel); +REGISTER_OP_CPU_KERNEL( + transpose_grad, + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.cu b/paddle/operators/transpose_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..af3f581462c919bbd2dd1067e536cc638f9c267d --- /dev/null +++ b/paddle/operators/transpose_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/transpose_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(transpose, + ops::TransposeKernel); +REGISTER_OP_GPU_KERNEL( + transpose_grad, + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.h b/paddle/operators/transpose_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ea299dce72ad340b0a65ee50582dc156b5ad7abb --- /dev/null +++ b/paddle/operators/transpose_op.h @@ -0,0 +1,128 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +void EigenTranspose(const framework::ExecutionContext& context, + const framework::Tensor& in, framework::Tensor& out, + std::vector axis) { + Eigen::array permute; + for (int i = 0; i < Rank; i++) { + permute[i] = axis[i]; + } + auto in_dim = in.dims(); + auto out_dim = out.dims(); + + auto eigen_in = framework::EigenTensor::From(in); + auto eigen_out = framework::EigenTensor::From(out); + auto& dev = context.GetEigenDevice(); + eigen_out.device(dev) = eigen_in.shuffle(permute); +} + +template +class TransposeKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + out->mutable_data(context.GetPlace()); + + std::vector axis = context.Attr>("axis"); + int ndims = axis.size(); + switch (ndims) { + case 1: + EigenTranspose(context, *x, *out, axis); + break; + case 2: + EigenTranspose(context, *x, *out, axis); + break; + case 3: + EigenTranspose(context, *x, *out, axis); + break; + case 4: + EigenTranspose(context, *x, *out, axis); + break; + case 5: + EigenTranspose(context, *x, *out, axis); + break; + case 6: + EigenTranspose(context, *x, *out, axis); + break; + default: + PADDLE_THROW("Tensors with rank at most 6 are supported"); + } + } +}; + +template +class TransposeGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* out_grad = + context.Input(framework::GradVarName("Out")); + auto* x_grad = + context.Output(framework::GradVarName("X")); + if (x_grad) { + x_grad->mutable_data(context.GetPlace()); + + std::vector axis = context.Attr>("axis"); + std::vector reversed_axis(axis); + + for (size_t i = 0; i < axis.size(); i++) { + reversed_axis[axis[i]] = i; + } + + int ndims = axis.size(); + + switch (ndims) { + case 1: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 2: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 3: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 4: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 5: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 6: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + default: + PADDLE_THROW("Tensors with rank at most 6 are supported"); + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index 184bcbc29c0d26a214345506f126f9cc0d406b07..17ea48361bc597ccfeb80884d51900e6567aa057 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -54,7 +54,7 @@ class UniformRandomOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(Attr("min") < Attr("max"), "uniform_random's min must less then max"); - auto* tensor = ctx.Output("Out"); + auto* tensor = ctx.Output("Out"); auto dims = Attr>("dims"); std::vector temp; temp.reserve(dims.size()); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 7c32eb0069f4075d72cd4c3654c83e3d5c98fb1c..0f57b81966647ca5c6f5cd2e5518d2d34942a549 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1565,6 +1565,10 @@ class LayerBase(object): self.config = g_config.model_config.layers.add() assert isinstance(self.config, LayerConfig) + use_mkldnn = bool(int(g_command_config_args.get("use_mkldnn", 0))) + mkldnn_acts = ['relu', 'tanh'] + if use_mkldnn and active_type in mkldnn_acts: + active_type = "mkldnn_" + active_type self.config.name = name self.config.type = type self.config.active_type = active_type diff --git a/python/paddle/v2/framework/op.py b/python/paddle/v2/framework/op.py index 6cca41e43b38b8cccb65ff9b347ef226dddecd4d..9086a5cc3452b178ec37fe6a3e358eaa4c5d606b 100644 --- a/python/paddle/v2/framework/op.py +++ b/python/paddle/v2/framework/op.py @@ -89,12 +89,16 @@ class OpDescCreationMethod(object): new_attr.f = user_defined_attr elif attr.type == framework_pb2.STRING: new_attr.s = user_defined_attr + elif attr.type == framework_pb2.BOOLEAN: + new_attr.b = user_defined_attr elif attr.type == framework_pb2.INTS: new_attr.ints.extend(user_defined_attr) elif attr.type == framework_pb2.FLOATS: new_attr.floats.extend(user_defined_attr) elif attr.type == framework_pb2.STRINGS: new_attr.strings.extend(user_defined_attr) + elif attr.type == framework_pb2.BOOLEANS: + new_attr.bools.extend(user_defined_attr) elif attr.type == framework_pb2.INT_PAIRS: for p in user_defined_attr: pair = new_attr.int_pairs.add() diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py new file mode 100644 index 0000000000000000000000000000000000000000..8f6d2be17758b7f6604d2db74fe466fb30695bd5 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -0,0 +1,223 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestExp(OpTest): + def setUp(self): + self.op_type = "exp" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.exp(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSigmoid(OpTest): + def setUp(self): + self.op_type = "sigmoid" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + +class TestTanh(OpTest): + def setUp(self): + self.op_type = "tanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.tanh(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSqrt(OpTest): + def setUp(self): + self.op_type = "sqrt" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.sqrt(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestAbs(OpTest): + def setUp(self): + self.op_type = "abs" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + # Because we set delta = 0.005 in caculating numeric gradient, + # if x is too small, such as 0.002, x_neg will be -0.003 + # x_pos will be 0.007, so the numeric gradient is unaccurate. + # we should avoid this + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.abs(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestRelu(OpTest): + def setUp(self): + self.op_type = "relu" + x = np.random.uniform(-1, 1, [11, 17]).astype("float32") + # The same reason with TestAbs + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.maximum(self.inputs['X'], 0)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestBRelu(OpTest): + def setUp(self): + self.op_type = "brelu" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + t_min = 1 + t_max = 4 + # The same with TestAbs + x[np.abs(x - t_min) < 0.005] = t_min + 0.02 + x[np.abs(x - t_max) < 0.005] = t_max + 0.02 + + self.inputs = {'X': x} + self.attrs = {'t_min': t_min, 't_max': t_max} + t = np.copy(x) + t[t < t_min] = t_min + t[t > t_max] = t_max + self.outputs = {'Y': t} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSoftRelu(OpTest): + def setUp(self): + self.op_type = "soft_relu" + x = np.random.uniform(-3, 3, [4, 4]).astype("float32") + threshold = 2 + # The same reason with TestAbs + x[np.abs(x - threshold) < 0.005] = threshold + 0.02 + x[np.abs(x + threshold) < 0.005] = -threshold + 0.02 + self.inputs = {'X': x} + self.attrs = {'threshold': threshold} + t = np.copy(x) + t[t < -threshold] = -threshold + t[t > threshold] = threshold + self.outputs = {'Y': np.log((np.exp(t) + 1))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestReciprocal(OpTest): + def setUp(self): + self.op_type = "reciprocal" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.outputs = {'Y': np.reciprocal(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.01) + + +class TestLog(OpTest): + def setUp(self): + self.op_type = "log" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.log(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSquare(OpTest): + def setUp(self): + self.op_type = "square" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.square(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestPow(OpTest): + def setUp(self): + self.op_type = "pow" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.attrs = {'factor': 3} + self.outputs = {'Y': np.power(self.inputs['X'], 3)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSTanh(OpTest): + def setUp(self): + self.op_type = "stanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + scale_a = 2.0 / 3.0 + scale_b = 1.7159 + self.attrs = {'scale_a': scale_a, 'scale_b': scale_b} + self.outputs = {'Y': scale_b * np.tanh(self.inputs['X'] * scale_a)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_clip_op.py b/python/paddle/v2/framework/tests/test_clip_op.py new file mode 100644 index 0000000000000000000000000000000000000000..5df6a494989017bab0416e0af962b2a85db046ba --- /dev/null +++ b/python/paddle/v2/framework/tests/test_clip_op.py @@ -0,0 +1,58 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestClipOp(OpTest): + def setUp(self): + self.max_relative_error = 0.006 + self.initTestCase() + input = np.random.random(self.shape).astype("float32") + input[np.abs(input - self.min) < self.max_relative_error] = 0.5 + input[np.abs(input - self.max) < self.max_relative_error] = 0.5 + self.op_type = "clip" + self.inputs = {'X': input, } + self.attrs = {} + self.attrs['min'] = self.min + self.attrs['max'] = self.max + self.outputs = { + 'Out': np.clip(self.inputs['X'], self.attrs['min'], + self.attrs['max']) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad( + ['X'], 'Out', max_relative_error=self.max_relative_error) + + def initTestCase(self): + self.shape = (4, 4) + self.max = 0.7 + self.min = 0.1 + + +class TestCase1(TestClipOp): + def initTestCase(self): + self.shape = (8, 16, 8) + self.max = 0.7 + self.min = 0 + + +class TestCase2(TestClipOp): + def initTestCase(self): + self.shape = (8, 16) + self.max = 1 + self.min = 0 + + +class TestCase3(TestClipOp): + def initTestCase(self): + self.shape = (4, 8, 16) + self.max = 0.7 + self.min = 0.2 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv2d_op.py b/python/paddle/v2/framework/tests/test_conv2d_op.py new file mode 100644 index 0000000000000000000000000000000000000000..118a5fc1cde5f4a908b065d581956e0855d50a52 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conv2d_op.py @@ -0,0 +1,103 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestConv2dOp(OpTest): + def setUp(self): + self.init_groups() + self.op_type = "conv2d" + batch_size = 2 + input_channels = 3 + input_height = 5 + input_width = 5 + output_channels = 6 + filter_height = 3 + filter_width = 3 + stride = 1 + padding = 0 + output_height = (input_height - filter_height + 2 * padding + ) / stride + 1 + output_width = (input_width - filter_width + 2 * padding) / stride + 1 + input = np.random.random((batch_size, input_channels, input_height, + input_width)).astype("float32") + + filter = np.random.random( + (output_channels, input_channels / self.groups, filter_height, + filter_width)).astype("float32") + output = np.ndarray( + (batch_size, output_channels, output_height, output_width)) + + self.inputs = {'Input': input, 'Filter': filter} + self.attrs = { + 'strides': [1, 1], + 'paddings': [0, 0], + 'groups': self.groups + } + + output_group_channels = output_channels / self.groups + input_group_channels = input_channels / self.groups + for batchid in xrange(batch_size): + for group in xrange(self.groups): + for outchannelid in range(group * output_group_channels, + (group + 1) * output_group_channels): + for rowid in xrange(output_height): + for colid in xrange(output_width): + start_h = (rowid * stride) - padding + start_w = (colid * stride) - padding + output_value = 0.0 + for inchannelid in range( + group * input_group_channels, + (group + 1) * input_group_channels): + for frowid in xrange(filter_height): + for fcolid in xrange(filter_width): + input_value = 0.0 + inrowid = start_h + frowid + incolid = start_w + fcolid + if ((inrowid >= 0 and + inrowid < input_height) and + (incolid >= 0 and + incolid < input_width)): + input_value = input[batchid][ + inchannelid][inrowid][incolid] + filter_value = filter[outchannelid][ + inchannelid % input_group_channels][ + frowid][fcolid] + output_value += input_value * filter_value + output[batchid][outchannelid][rowid][ + colid] = output_value + + self.outputs = {'Output': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.05) + + def test_check_grad_no_filter(self): + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.05, + no_grad_set=set(['Filter'])) + + def test_check_grad_no_input(self): + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.05, + no_grad_set=set(['Input'])) + + def init_groups(self): + self.groups = 1 + + +class TestWithGroup(TestConv2dOp): + def init_groups(self): + self.groups = 3 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_cos_sim_op.py b/python/paddle/v2/framework/tests/test_cos_sim_op.py index d314ce391ea2f10a8bd77c24e84fa3e1eebb6c73..47557ccb41d1e835b5d04d1b94f54dfc7aa2855a 100644 --- a/python/paddle/v2/framework/tests/test_cos_sim_op.py +++ b/python/paddle/v2/framework/tests/test_cos_sim_op.py @@ -24,15 +24,15 @@ class TestCosSimOp(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.05) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.06) def test_check_grad_ingore_x(self): self.check_grad( - ['Y'], 'Out', max_relative_error=0.05, no_grad_set=set("X")) + ['Y'], 'Out', max_relative_error=0.06, no_grad_set=set("X")) def test_check_grad_ingore_y(self): self.check_grad( - ['X'], 'Out', max_relative_error=0.05, no_grad_set=set('Y')) + ['X'], 'Out', max_relative_error=0.06, no_grad_set=set('Y')) class TestCosSimOp2(TestCosSimOp): diff --git a/python/paddle/v2/framework/tests/test_crop_op.py b/python/paddle/v2/framework/tests/test_crop_op.py new file mode 100644 index 0000000000000000000000000000000000000000..62c883bdc130021d06c33ded9c2865505da0b719 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_crop_op.py @@ -0,0 +1,91 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def crop(data, offsets, crop_shape): + def indexOf(shape, index): + result = [] + for dim in reversed(shape): + result.append(index % dim) + index = index / dim + return result[::-1] + + result = [] + for i, value in enumerate(data.flatten()): + index = indexOf(data.shape, i) + selected = True + if len(index) == len(offsets): + for j, offset in enumerate(offsets): + selected = selected and index[j] >= offset and index[ + j] < crop_shape[j] + offset + if selected: + result.append(value) + return np.array(result).reshape(crop_shape) + + +class TestCropOp(OpTest): + def setUp(self): + self.op_type = "crop" + self.crop_by_input = False + self.attrs = {} + self.initTestCase() + self.attrs['offsets'] = self.offsets + if self.crop_by_input: + self.inputs = { + 'X': np.random.random(self.x_shape).astype("float32"), + 'Y': np.random.random(self.crop_shape).astype("float32") + } + else: + self.attrs['shape'] = self.crop_shape + self.inputs = { + 'X': np.random.random(self.x_shape).astype("float32"), + } + self.outputs = { + 'Out': crop(self.inputs['X'], self.offsets, self.crop_shape) + } + + def initTestCase(self): + self.x_shape = (8, 8) + self.crop_shape = (2, 2) + self.offsets = [1, 2] + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X'], 'Out', max_relative_error=0.006) + + +class TestCase1(TestCropOp): + def initTestCase(self): + self.x_shape = (16, 8, 32) + self.crop_shape = [2, 2, 3] + self.offsets = [1, 5, 3] + + +class TestCase2(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 8) + self.crop_shape = [4, 8] + self.offsets = [0, 0] + + +class TestCase3(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 8, 16) + self.crop_shape = [2, 2, 3] + self.offsets = [1, 5, 3] + self.crop_by_input = True + + +class TestCase4(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 4) + self.crop_shape = [4, 4] + self.offsets = [0, 0] + self.crop_by_input = True + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_cross_entropy_op.py index 5cccfc6ade031ca95911ac912a79d00c5f4cef3a..8fdf622970089f3504a30d2a9967cadf821bb87e 100644 --- a/python/paddle/v2/framework/tests/test_cross_entropy_op.py +++ b/python/paddle/v2/framework/tests/test_cross_entropy_op.py @@ -19,7 +19,7 @@ class TestCrossEntropyOp1(OpTest): dtype="float32") self.inputs = {"X": X, "Label": label} self.outputs = {"Y": cross_entropy} - self.attrs = {"soft_label": 0} + self.attrs = {'soft_label': False} def test_check_output(self): self.check_output() @@ -43,10 +43,9 @@ class TestCrossEntropyOp2(OpTest): label /= label.sum(axis=1, keepdims=True) cross_entropy = (-label * np.log(X)).sum( axis=1, keepdims=True).astype("float32") - - self.inputs = {"X": X, "Label": label} - self.outputs = {"Y": cross_entropy} - self.attrs = {"soft_label": 1} + self.inputs = {'X': X, 'Label': label} + self.outputs = {'Y': cross_entropy} + self.attrs = {'soft_label': True} def test_check_output(self): self.check_output() @@ -75,9 +74,9 @@ class TestCrossEntropyOp3(OpTest): dtype="float32") cross_entropy2 = (-label * np.log(X)).sum( axis=1, keepdims=True).astype("float32") - self.inputs = {"X": X, "Label": label} - self.outputs = {"Y": cross_entropy} - self.attrs = {"soft_label": 1} + self.inputs = {'X': X, 'Label': label} + self.outputs = {'Y': cross_entropy} + self.attrs = {'soft_label': True} def test_check_output(self): self.check_output() diff --git a/python/paddle/v2/framework/tests/test_dropout_op.py b/python/paddle/v2/framework/tests/test_dropout_op.py index 3638fee1a1c26195791bc1f5a46dd749da0aee95..29fc702791184aaacf335e13bcc6d03082bb49a6 100644 --- a/python/paddle/v2/framework/tests/test_dropout_op.py +++ b/python/paddle/v2/framework/tests/test_dropout_op.py @@ -7,7 +7,7 @@ class TestDropoutOp(OpTest): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} - self.attrs = {'dropout_prob': 0.0, 'is_training': 1} + self.attrs = {'dropout_prob': 0.0, 'is_training': True} self.outputs = {'Out': self.inputs['X'], 'Mask': np.ones((32, 64))} def test_check_output(self): @@ -21,7 +21,7 @@ class TestDropoutOp2(TestDropoutOp): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} - self.attrs = {'dropout_prob': 1.0, 'is_training': 1} + self.attrs = {'dropout_prob': 1.0, 'is_training': True} self.outputs = {'Out': np.zeros((32, 64)), 'Mask': np.zeros((32, 64))} @@ -29,7 +29,7 @@ class TestDropoutOp3(TestDropoutOp): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} - self.attrs = {'dropout_prob': 0.0, 'is_training': 1} + self.attrs = {'dropout_prob': 0.0, 'is_training': True} self.outputs = {'Out': self.inputs['X'], 'Mask': np.ones((32, 64, 2))} @@ -37,7 +37,7 @@ class TestDropoutOp4(OpTest): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64)).astype("float32")} - self.attrs = {'dropout_prob': 0.35, 'is_training': 0} + self.attrs = {'dropout_prob': 0.35, 'is_training': False} self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} def test_check_output(self): @@ -48,7 +48,7 @@ class TestDropoutOp5(OpTest): def setUp(self): self.op_type = "dropout" self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} - self.attrs = {'dropout_prob': 0.75, 'is_training': 0} + self.attrs = {'dropout_prob': 0.75, 'is_training': False} self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} def test_check_output(self): diff --git a/python/paddle/v2/framework/tests/test_elementwise_add_op.py b/python/paddle/v2/framework/tests/test_elementwise_add_op.py new file mode 100644 index 0000000000000000000000000000000000000000..f3101a709b8bcf58e8682ab3d0ca5217a7f3572d --- /dev/null +++ b/python/paddle/v2/framework/tests/test_elementwise_add_op.py @@ -0,0 +1,96 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestElementwiseOp(OpTest): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float32") + } + self.outputs = {'Out': np.add(self.inputs['X'], self.inputs['Y'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.005) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) + + +class TestElementwiseAddOp_Vector(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.random((32, )).astype("float32"), + 'Y': np.random.random((32, )).astype("float32") + } + self.outputs = {'Out': np.add(self.inputs['X'], self.inputs['Y'])} + + +class TestElementwiseAddOp_broadcast_0(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(2).astype(np.float32) + } + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': self.inputs['X'] + self.inputs['Y'].reshape(2, 1, 1) + } + + +class TestElementwiseAddOp_broadcast_1(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(3).astype(np.float32) + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': self.inputs['X'] + self.inputs['Y'].reshape(1, 3, 1) + } + + +class TestElementwiseAddOp_broadcast_2(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(4).astype(np.float32) + } + + self.outputs = { + 'Out': self.inputs['X'] + self.inputs['Y'].reshape(1, 1, 4) + } + + +class TestElementwiseAddOp_broadcast_3(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4, 5).astype(np.float32), + 'Y': np.random.rand(3, 4).astype(np.float32) + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': self.inputs['X'] + self.inputs['Y'].reshape(1, 3, 4, 1) + } + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_elementwise_div_op.py b/python/paddle/v2/framework/tests/test_elementwise_div_op.py new file mode 100644 index 0000000000000000000000000000000000000000..41cb2b7767eb8e01e46e770a5da21b609f4eb911 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_elementwise_div_op.py @@ -0,0 +1,105 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class ElementwiseDivOp(OpTest): + def setUp(self): + self.op_type = "elementwise_div" + """ Warning + CPU gradient check error! + 'X': np.random.random((32,84)).astype("float32"), + 'Y': np.random.random((32,84)).astype("float32") + """ + self.inputs = { + 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float32") + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.05) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.05, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.05, no_grad_set=set('Y')) + + +class TestElementwiseDivOp_Vector(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [32]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [32]).astype("float32") + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestElementwiseDivOp_broadcast_0(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [2]).astype("float32") + } + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(2, 1, 1)) + } + + +class TestElementwiseDivOp_broadcast_1(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [3]).astype("float32") + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 1)) + } + + +class TestElementwiseDivOp_broadcast_2(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [4]).astype("float32") + } + + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 4)) + } + + +class TestElementwiseDivOp_broadcast_3(ElementwiseDivOp): + def setUp(self): + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4, 5]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [3, 4]).astype("float32") + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 3, 4, 1)) + } + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_elementwise_mul_op.py b/python/paddle/v2/framework/tests/test_elementwise_mul_op.py index e268cfddb26721a35ddd2d2cc18f526ff7b2f6d9..cee4385a8176f7a441a280e3cd40c39ca51493c5 100644 --- a/python/paddle/v2/framework/tests/test_elementwise_mul_op.py +++ b/python/paddle/v2/framework/tests/test_elementwise_mul_op.py @@ -3,14 +3,9 @@ import numpy as np from op_test import OpTest -class TestElementwiseMulOp_Matrix(OpTest): +class ElementwiseMulOp(OpTest): def setUp(self): self.op_type = "elementwise_mul" - """ Warning - CPU gradient check error! - 'X': np.random.random((32,84)).astype("float32"), - 'Y': np.random.random((32,84)).astype("float32") - """ self.inputs = { 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float32"), 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float32") @@ -32,7 +27,7 @@ class TestElementwiseMulOp_Matrix(OpTest): ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) -class TestElementwiseMulOp_Vector(OpTest): +class TestElementwiseMulOp_Vector(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { @@ -41,22 +36,8 @@ class TestElementwiseMulOp_Vector(OpTest): } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} - def test_check_output(self): - self.check_output() - - def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.1) - - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.1, no_grad_set=set("X")) - - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) - -class TestElementwiseMulOp_broadcast_0(OpTest): +class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { @@ -69,22 +50,8 @@ class TestElementwiseMulOp_broadcast_0(OpTest): 'Out': self.inputs['X'] * self.inputs['Y'].reshape(2, 1, 1) } - def test_check_output(self): - self.check_output() - - def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.1) - - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.1, no_grad_set=set("X")) - - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) - -class TestElementwiseMulOp_broadcast_1(OpTest): +class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { @@ -97,22 +64,8 @@ class TestElementwiseMulOp_broadcast_1(OpTest): 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 3, 1) } - def test_check_output(self): - self.check_output() - - def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.1) - - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.1, no_grad_set=set("X")) - - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) - -class TestElementwiseMulOp_broadcast_2(OpTest): +class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { @@ -124,22 +77,8 @@ class TestElementwiseMulOp_broadcast_2(OpTest): 'Out': self.inputs['X'] * self.inputs['Y'].reshape(1, 1, 4) } - def test_check_output(self): - self.check_output() - - def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.1) - - def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.1, no_grad_set=set("X")) - - def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) - -class TestElementwiseMulOp_broadcast_3(OpTest): +class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { diff --git a/python/paddle/v2/framework/tests/test_elementwise_sub_op.py b/python/paddle/v2/framework/tests/test_elementwise_sub_op.py new file mode 100644 index 0000000000000000000000000000000000000000..be982e8c57b30b91c2834bd5db38ea3c89f573ee --- /dev/null +++ b/python/paddle/v2/framework/tests/test_elementwise_sub_op.py @@ -0,0 +1,96 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestElementwiseOp(OpTest): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float32") + } + self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.005) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) + + +class TestElementwiseSubOp_Vector(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.random((32, )).astype("float32"), + 'Y': np.random.random((32, )).astype("float32") + } + self.outputs = {'Out': self.inputs['X'] - self.inputs['Y']} + + +class TestElementwiseSubOp_broadcast_0(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(2).astype(np.float32) + } + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(2, 1, 1) + } + + +class TestElementwiseSubOp_broadcast_1(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(3).astype(np.float32) + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 3, 1) + } + + +class TestElementwiseSubOp_broadcast_2(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(4).astype(np.float32) + } + + self.outputs = { + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 1, 4) + } + + +class TestElementwiseSubOp_broadcast_3(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_sub" + self.inputs = { + 'X': np.random.rand(2, 3, 4, 5).astype(np.float32), + 'Y': np.random.rand(3, 4).astype(np.float32) + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': self.inputs['X'] - self.inputs['Y'].reshape(1, 3, 4, 1) + } + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_fill_zeros_like_op.py b/python/paddle/v2/framework/tests/test_fill_zeros_like_op.py index 2473daaba24438819f3f55ccc40fe1c64ee59960..eff8fa87d9c0dafc6935604101e94ee6c8b081ce 100644 --- a/python/paddle/v2/framework/tests/test_fill_zeros_like_op.py +++ b/python/paddle/v2/framework/tests/test_fill_zeros_like_op.py @@ -6,8 +6,8 @@ from op_test import OpTest class TestFillZerosLikeOp(OpTest): def setUp(self): self.op_type = "fill_zeros_like" - self.inputs = {'Src': np.random.random((219, 232)).astype("float32")} - self.outputs = {'Dst': np.zeros_like(self.inputs["Src"])} + self.inputs = {'X': np.random.random((219, 232)).astype("float32")} + self.outputs = {'Y': np.zeros_like(self.inputs["X"])} def test_check_output(self): self.check_output() diff --git a/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e2b57529b0723b4ab18b73801cd2816d8025dd --- /dev/null +++ b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py @@ -0,0 +1,39 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def modified_huber_loss_forward(val): + if val < -1: + return -4 * val + elif val < 1: + return (1 - val) * (1 - val) + else: + return 0 + + +class TestModifiedHuberLossOp(OpTest): + def setUp(self): + self.op_type = 'modified_huber_loss' + samples_num = 32 + self.inputs = { + 'X': np.random.uniform(-1, 1., (samples_num, 1)).astype('float32'), + 'Y': np.random.choice([0, 1], samples_num).reshape((samples_num, 1)) + } + product_res = self.inputs['X'] * (2 * self.inputs['Y'] - 1) + loss = np.vectorize(modified_huber_loss_forward)(product_res) + + self.outputs = { + 'IntermediateVal': product_res, + 'Out': loss.reshape((samples_num, 1)) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out', max_relative_error=0.005) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rank_loss_op.py b/python/paddle/v2/framework/tests/test_rank_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..0e41ab1b3fd8fa8b62c5f3b914b752918119a265 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_rank_loss_op.py @@ -0,0 +1,32 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestRankLossOp(OpTest): + def setUp(self): + self.op_type = "rank_loss" + batch_size = 5 + # labels_{i} = {0, 1.0} or {0, 0.5, 1.0} + label = np.random.randint(0, 2, size=(batch_size, 1)).astype("float32") + left = np.random.random((batch_size, 1)).astype("float32") + right = np.random.random((batch_size, 1)).astype("float32") + loss = np.log(1.0 + np.exp(left - right)) - label * (left - right) + self.inputs = {'Label': label, 'Left': left, 'Right': right} + self.outputs = {'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["Left", "Right"], "Out") + + def test_check_grad_ignore_left(self): + self.check_grad(["Right"], "Out", no_grad_set=set('Left')) + + def test_check_grad_ignore_right(self): + self.check_grad(["Left"], "Out", no_grad_set=set('Right')) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 22e680fd783ec681e95326fb84db34570265cffc..79eda70021b76cd06e4c40740b1ca49476f4c503 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -59,7 +59,6 @@ class PySimpleRNNTest(unittest.TestCase): def test_forward(self): output = self.rnn.forward() - print 'output', output def create_tensor(scope, name, shape, np_data): @@ -103,7 +102,7 @@ class TestRecurrentOp(unittest.TestCase): ctx = core.DeviceContext.create(core.CPUPlace()) self.rnnop.infer_shape(self.scope) self.rnnop.run(self.scope, ctx) - return np.array(self.scope.find_var("h").get_tensor()) + return np.array(self.scope.find_var("h@mem").get_tensor()) def create_global_variables(self): # create inlink @@ -123,8 +122,7 @@ class TestRecurrentOp(unittest.TestCase): create_tensor(self.scope, "h_boot", [self.batch_size, self.input_dim], h_boot_np_data) self.scope.new_var("step_scopes") - self.scope.new_var("h@alias") - self.scope.new_var("h") + self.scope.new_var("h@mem") def create_rnn_op(self): # create RNNOp @@ -134,20 +132,18 @@ class TestRecurrentOp(unittest.TestCase): boot_memories=["h_boot"], step_net="stepnet", # outputs - outlinks=["h"], + outlinks=["h@mem"], step_scopes="step_scopes", # attributes - inlink_alias=["x@alias"], - outlink_alias=["h@alias"], pre_memories=["h@pre"], - memories=["h@alias"]) + memories=["h@mem"]) def create_step_net(self): stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") + x_fc_op = Operator("mul", X="x", Y="W", Out="Wx") h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") sum_op = Operator("add", X="Wx", Y="Uh", Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@alias") + sig_op = Operator("sigmoid", X="sum", Y="h@mem") for op in [x_fc_op, h_fc_op, sum_op, sig_op]: stepnet.append_op(op) diff --git a/python/paddle/v2/framework/tests/test_sigmoid_op.py b/python/paddle/v2/framework/tests/test_sigmoid_op.py deleted file mode 100644 index d65d887db4af58c40e4e78fdbfd8e8ee668b7ee3..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_sigmoid_op.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestSigmoidOp(OpTest): - def setUp(self): - self.op_type = "sigmoid" - self.inputs = { - 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") - } - self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Y", max_relative_error=0.007) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..be940327ec910ccb9de59d45029513ff4779443b --- /dev/null +++ b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py @@ -0,0 +1,87 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def smooth_l1_loss_forward(val, sigma2): + abs_val = abs(val) + if abs_val < 1.0 / sigma2: + return 0.5 * val * val * sigma2 + else: + return abs_val - 0.5 / sigma2 + + +class TestSmoothL1LossOp1(OpTest): + def setUp(self): + self.op_type = "smooth_l1_loss" + dims = (5, 10) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2).sum(1) + loss = loss.reshape((dims[0], 1)) + self.outputs = {'Diff': diff, 'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.02) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.03, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.03, no_grad_set=set('Y')) + + +class TestSmoothL1LossOp2(OpTest): + def setUp(self): + self.op_type = "smooth_l1_loss" + dims = (5, 10) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32"), + 'InsideWeight': np.random.random(dims).astype("float32"), + 'OutsideWeight': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + diff = diff * self.inputs['InsideWeight'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2) + loss = loss * self.inputs['OutsideWeight'] + loss = loss.sum(1).reshape((dims[0], 1)) + self.outputs = {'Diff': diff, 'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.03) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight'])) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_transpose_op.py b/python/paddle/v2/framework/tests/test_transpose_op.py new file mode 100644 index 0000000000000000000000000000000000000000..9409cbaa00f792b60d5950556b869108aa732478 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_transpose_op.py @@ -0,0 +1,56 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestTransposeOp(OpTest): + def setUp(self): + self.initTestCase() + self.op_type = "transpose" + self.inputs = {'X': np.random.random(self.shape).astype("float32")} + self.attrs = {'axis': list(self.axis)} + self.outputs = {'Out': self.inputs['X'].transpose(self.axis)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + def initTestCase(self): + self.shape = (3, 4) + self.axis = (1, 0) + + +class TestCase0(TestTransposeOp): + def initTestCase(self): + self.shape = (3, ) + self.axis = (0, ) + + +class TestCase1(TestTransposeOp): + def initTestCase(self): + self.shape = (3, 4, 5) + self.axis = (0, 2, 1) + + +class TestCase2(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5) + self.axis = (0, 2, 3, 1) + + +class TestCase3(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5, 6) + self.axis = (4, 2, 3, 1, 0) + + +class TestCase4(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5, 6, 1) + self.axis = (4, 2, 3, 1, 0, 5) + + +if __name__ == '__main__': + unittest.main()