diff --git a/.travis.yml b/.travis.yml index d0e2696f100e55f320e410afd6a3038db647f76f..c51e02eb79a9e53a2b8d1d663e8f0c3e0d8c3a61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ addons: - automake - libtool - ccache + ssh_known_hosts: 52.76.173.135 before_install: - if [[ "$JOB" == "check_style" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python @@ -42,6 +43,14 @@ script: - | timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi; + - | + if [[ "$JOB" != "build_doc" ]]; then exit 0; fi; + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi; + if [[ "$TRAVIS_BRANCH" != "develop" && ! "$TRAVIS_BRANCH" =~ ^v[[:digit:]]+\.[[:digit:]]+(\.[[:digit:]]+)?(-\S*)?$ ]]; then exit 0; fi; + export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/master/scripts/deploy/deploy_docs.sh + export DOCS_DIR=`pwd` + cd .. + curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH $DOCS_DIR $DOCS_DIR/build/doc notifications: email: on_success: change diff --git a/CMakeLists.txt b/CMakeLists.txt index 264420ad830ed39b38f1918951d8d66c84fd5ee9..fd3582a1bca199d62d19550ffdd1efe9db520fa7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,7 @@ include(external/swig) # download, build, install swig include(external/warpctc) # download, build, install warpctc include(external/any) # download libn::any include(external/eigen) # download eigen3 -include(external/pybind11) # download pybind11 +include(external/pybind11) # download pybind11 include(external/nccl) include(cudnn) # set cudnn libraries, must before configure diff --git a/benchmark/IntelOptimizedPaddle.md b/benchmark/IntelOptimizedPaddle.md new file mode 100644 index 0000000000000000000000000000000000000000..040f5ffa41968cbf93a817faa1db86c18956341e --- /dev/null +++ b/benchmark/IntelOptimizedPaddle.md @@ -0,0 +1,48 @@ +# Benchmark + +Machine: + +- Server + - Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz, 2 Sockets, 20 Cores per socket +- Laptop + - DELL XPS15-9560-R1745: i7-7700HQ 8G 256GSSD + - i5 MacBook Pro (Retina, 13-inch, Early 2015) +- Desktop + - i7-6700k + +System: CentOS release 6.3 (Final), Docker 1.12.1. + +PaddlePaddle: paddlepaddle/paddle:latest (TODO: will rerun after 0.11.0) + +- MKL-DNN tag v0.10 +- MKLML 2018.0.20170720 +- OpenBLAS v0.2.20 + +On each machine, we will test and compare the performance of training on single node using MKL-DNN / MKLML / OpenBLAS respectively. + +## Benchmark Model + +### Server +Test on batch size 64, 128, 256 on Intel(R) Xeon(R) Gold 6148 CPU @ 2.40GHz + +Input image size - 3 * 224 * 224, Time: images/second + +- VGG-19 + +| BatchSize | 64 | 128 | 256 | +|--------------|-------| -----| --------| +| OpenBLAS | 7.82 | 8.62 | 10.34 | +| MKLML | 11.02 | 12.86 | 15.33 | +| MKL-DNN | 27.69 | 28.8 | 29.27 | + + +chart on batch size 128 +TBD + + - ResNet + - GoogLeNet + +### Laptop +TBD +### Desktop +TBD diff --git a/benchmark/paddle/image/resnet.py b/benchmark/paddle/image/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..6ae1857642e8df4b3859eec68a3a5227d1c4fcb3 --- /dev/null +++ b/benchmark/paddle/image/resnet.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +from paddle.trainer_config_helpers import * + +height = 224 +width = 224 +num_class = 1000 +batch_size = get_config_arg('batch_size', int, 64) +layer_num = get_config_arg("layer_num", int, 50) +is_test = get_config_arg("is_test", bool, False) + +args = {'height': height, 'width': width, 'color': True, 'num_class': num_class} +define_py_data_sources2( + "train.list", None, module="provider", obj="process", args=args) + +settings( + batch_size=batch_size, + learning_rate=0.01 / batch_size, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * batch_size)) + + +#######################Network Configuration ############# +def conv_bn_layer(name, + input, + filter_size, + num_filters, + stride, + padding, + channels=None, + active_type=ReluActivation()): + """ + A wrapper for conv layer with batch normalization layers. + Note: + conv layer has no activation. + """ + + tmp = img_conv_layer( + name=name + "_conv", + input=input, + filter_size=filter_size, + num_channels=channels, + num_filters=num_filters, + stride=stride, + padding=padding, + act=LinearActivation(), + bias_attr=False) + return batch_norm_layer( + name=name + "_bn", input=tmp, act=active_type, use_global_stats=is_test) + + +def bottleneck_block(name, input, num_filters1, num_filters2): + """ + A wrapper for bottlenect building block in ResNet. + Last conv_bn_layer has no activation. + Addto layer has activation of relu. + """ + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=1, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[input, last_name], act=ReluActivation()) + + +def mid_projection(name, input, num_filters1, num_filters2, stride=2): + """ + A wrapper for middile projection in ResNet. + projection shortcuts are used for increasing dimensions, + and other shortcuts are identity + branch1: projection shortcuts are used for increasing + dimensions, has no activation. + branch2x: bottleneck building block, shortcuts are identity. + """ + # stride = 2 + branch1 = conv_bn_layer( + name=name + '_branch1', + input=input, + filter_size=1, + num_filters=num_filters2, + stride=stride, + padding=0, + active_type=LinearActivation()) + + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=stride, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[branch1, last_name], act=ReluActivation()) + + +img = data_layer(name='image', size=height * width * 3) + + +def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): + """ + A wrapper for 50,101,152 layers of ResNet. + res2_num: number of blocks stacked in conv2_x + res3_num: number of blocks stacked in conv3_x + res4_num: number of blocks stacked in conv4_x + res5_num: number of blocks stacked in conv5_x + """ + # For ImageNet + # conv1: 112x112 + tmp = conv_bn_layer( + "conv1", + input=img, + filter_size=7, + channels=3, + num_filters=64, + stride=2, + padding=3) + tmp = img_pool_layer(name="pool1", input=tmp, pool_size=3, stride=2) + + # conv2_x: 56x56 + tmp = mid_projection( + name="res2_1", input=tmp, num_filters1=64, num_filters2=256, stride=1) + for i in xrange(2, res2_num + 1, 1): + tmp = bottleneck_block( + name="res2_" + str(i), input=tmp, num_filters1=64, num_filters2=256) + + # conv3_x: 28x28 + tmp = mid_projection( + name="res3_1", input=tmp, num_filters1=128, num_filters2=512) + for i in xrange(2, res3_num + 1, 1): + tmp = bottleneck_block( + name="res3_" + str(i), + input=tmp, + num_filters1=128, + num_filters2=512) + + # conv4_x: 14x14 + tmp = mid_projection( + name="res4_1", input=tmp, num_filters1=256, num_filters2=1024) + for i in xrange(2, res4_num + 1, 1): + tmp = bottleneck_block( + name="res4_" + str(i), + input=tmp, + num_filters1=256, + num_filters2=1024) + + # conv5_x: 7x7 + tmp = mid_projection( + name="res5_1", input=tmp, num_filters1=512, num_filters2=2048) + for i in xrange(2, res5_num + 1, 1): + tmp = bottleneck_block( + name="res5_" + str(i), + input=tmp, + num_filters1=512, + num_filters2=2048) + + tmp = img_pool_layer( + name='avgpool', + input=tmp, + pool_size=7, + stride=1, + pool_type=AvgPooling()) + + return fc_layer(input=tmp, size=num_class, act=SoftmaxActivation()) + + +if layer_num == 50: + resnet = deep_res_net(3, 4, 6, 3) +elif layer_num == 101: + resnet = deep_res_net(3, 4, 23, 3) +elif layer_num == 152: + resnet = deep_res_net(3, 8, 36, 3) +else: + print("Wrong layer number.") + +lbl = data_layer(name="label", size=num_class) +loss = cross_entropy(name='loss', input=resnet, label=lbl) +inputs(img, lbl) +outputs(loss) diff --git a/benchmark/paddle/image/run_mkldnn.sh b/benchmark/paddle/image/run_mkldnn.sh index e31fec1cd850157d90ddcab2d559d52381ecd317..a4527e04968cf8c8c3c31d16f50bc3e28381f6d8 100755 --- a/benchmark/paddle/image/run_mkldnn.sh +++ b/benchmark/paddle/image/run_mkldnn.sh @@ -5,22 +5,23 @@ function train() { export OMP_DYNAMIC="FALSE" export KMP_AFFINITY="granularity=fine,compact,0,0" topology=$1 - bs=$2 - use_mkldnn=$3 - if [ $3 == "True" ]; then + layer_num=$2 + bs=$3 + use_mkldnn=$4 + if [ $4 == "True" ]; then thread=1 - log="logs/${topology}-mkldnn-${bs}.log" - elif [ $3 == "False" ]; then + log="logs/${topology}-${layer_num}-mkldnn-${bs}.log" + elif [ $4 == "False" ]; then thread=`nproc` # each trainer_count use only 1 core to avoid conflict export OMP_NUM_THREADS=1 export MKL_NUM_THREADS=1 - log="logs/${topology}-${thread}mklml-${bs}.log" + log="logs/${topology}-${layer_num}-${thread}mklml-${bs}.log" else echo "Wrong input $3, use True or False." exit 0 fi - args="batch_size=${bs}" + args="batch_size=${bs},layer_num=${layer_num}" config="${topology}.py" paddle train --job=time \ --config=$config \ @@ -40,12 +41,9 @@ if [ ! -d "logs" ]; then mkdir logs fi -#========== mkldnn ==========# -train vgg 64 True -train vgg 128 True -train vgg 256 True - -#========== mklml ===========# -train vgg 64 False -train vgg 128 False -train vgg 256 False +for use_mkldnn in True False; do + for batchsize in 64 128 256; do + train vgg 19 $batchsize $use_mkldnn + train resnet 50 $batchsize $use_mkldnn + done +done diff --git a/benchmark/paddle/image/vgg.py b/benchmark/paddle/image/vgg.py index b8429975f5c83df6996e71478fe276b246e8b77b..420884ed8e1ae36a3f1772bfbe8323f3d0ea71e6 100644 --- a/benchmark/paddle/image/vgg.py +++ b/benchmark/paddle/image/vgg.py @@ -13,7 +13,7 @@ define_py_data_sources2( settings( batch_size=batch_size, - learning_rate=0.01 / batch_size, + learning_rate=0.001 / batch_size, learning_method=MomentumOptimizer(0.9), regularization=L2Regularization(0.0005 * batch_size)) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 8fdc382f0c1c453a01dba884a3dad216e1c3092c..b21fc43904d9aafe9f7d019dfbe5b1c0d3f9e2d6 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -1,17 +1,12 @@ # Find the CBlas and lapack libraries # -# It will search MKL, atlas, OpenBlas, reference-cblas in order. +# It will search MKLML, atlas, OpenBlas, reference-cblas in order. # # If any cblas implementation found, the following variable will be set. -# CBLAS_PROVIDER # one of MKL, ATLAS, OPENBLAS, REFERENCE +# CBLAS_PROVIDER # one of MKLML, ATLAS, OPENBLAS, REFERENCE # CBLAS_INC_DIR # the include directory for cblas. # CBLAS_LIBS # a list of libraries should be linked by paddle. # # Each library should be full path to object file. -# -# User should set one of MKL_ROOT, ATLAS_ROOT, OPENBLAS_ROOT, REFERENCE_CBLAS_ROOT -# during cmake. If none of them set, it will try to find cblas implementation in -# system paths. -# set(CBLAS_FOUND OFF) @@ -30,44 +25,6 @@ if(WITH_MKLML AND MKLML_INC_DIR AND MKLML_LIB) return() endif() -## Then find MKL. -set(INTEL_MKL_ROOT "/opt/intel/mkl" CACHE PATH "Folder contains intel mkl libs") -set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains env MKL") - -set(MKL_INCLUDE_SEARCH_PATHS - ${MKL_ROOT}/include - ${INTEL_MKL_ROOT}/include) -set(MKL_LIB_SEARCH_PATHS - ${MKL_ROOT}/lib - ${MKL_ROOT}/lib/intel64 - ${INTEL_MKL_ROOT}/lib - ${INTEL_MKL_ROOT}/lib/intel64) - -find_path(MKL_INC_DIR mkl.h PATHS - ${MKL_INCLUDE_SEARCH_PATHS}) -find_path(MKL_LAPACK_INC_DIR mkl_lapacke.h PATHS - ${MKL_INCLUDE_SEARCH_PATHS}) -find_library(MKL_CORE_LIB NAMES mkl_core PATHS - ${MKL_LIB_SEARCH_PATHS}) -find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS - ${MKL_LIB_SEARCH_PATHS}) -find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS - ${MKL_LIB_SEARCH_PATHS}) - -if(MKL_LAPACK_INC_DIR AND MKL_INC_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) - set(CBLAS_FOUND ON) - set(CBLAS_PROVIDER MKL) - set(CBLAS_INC_DIR ${MKL_INC_DIR} ${MKL_LAPACK_INC_DIR}) - set(CBLAS_LIBRARIES ${MKL_INTEL_LP64} ${MKL_SEQUENTIAL_LIB} ${MKL_CORE_LIB}) - - add_definitions(-DPADDLE_USE_MKL) - add_definitions(-DLAPACK_FOUND) - - message(STATUS "Found MKL (include: ${MKL_INC_DIR}, library: ${CBLAS_LIBRARIES})") - message(STATUS "Found lapack in MKL (include: ${MKL_LAPACK_INC_DIR})") - return() -endif() - ## Then find atlas. set(ATLAS_ROOT $ENV{ATLAS_ROOT} CACHE PATH "Folder contains Atlas") set(ATLAS_INCLUDE_SEARCH_PATHS diff --git a/cmake/cross_compiling/ios.cmake b/cmake/cross_compiling/ios.cmake index 0b38943952f7fb9052368fe95eb31dd7592d8a47..310450f7d009dc0cdae9c0079a96445af8ec8f95 100644 --- a/cmake/cross_compiling/ios.cmake +++ b/cmake/cross_compiling/ios.cmake @@ -79,9 +79,8 @@ if(NOT DEFINED IOS_ARCH) # FIXME(liuyiqun): support "armv7;armv7s;arm64" future set(IOS_ARCH "arm64") elseif(IOS_PLATFORM STREQUAL "SIMULATOR") - set(IOS_ARCH "i386;x86_64") - elseif(IOS_PLATFORM STREQUAL "WATCHOS") - set(IOS_ARCH armv7k) + # FIXME(liuyiqun): support "i386;x86_64" future + set(IOS_ARCH "x86_64") endif() endif() set(CMAKE_OSX_ARCHITECTURES ${IOS_ARCH} CACHE string "Build architecture for iOS") diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 9686df00219001769d074ee815d9cc8db0258496..5a06825beb73e85d8a55b7b578b187bee2c4340c 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -46,16 +46,20 @@ IF(${CBLAS_PROVIDER} STREQUAL "MKLML") MESSAGE(STATUS "Build MKLDNN with ${MKLDNN_MKLROOT}") ENDIF() +SET(MKLDNN_CFLAG "${CMAKE_C_FLAGS} -Wno-error=strict-overflow") +SET(MKLDNN_CXXFLAG "${CMAKE_CXX_FLAGS} -Wno-error=strict-overflow") ExternalProject_Add( ${MKLDNN_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS ${MKLDNN_DEPENDS} GIT_REPOSITORY "https://github.com/01org/mkl-dnn.git" - GIT_TAG "v0.10" + GIT_TAG "v0.11" PREFIX ${MKLDNN_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${MKLDNN_INSTALL_DIR} CMAKE_ARGS -DMKLROOT=${MKLDNN_MKLROOT} + CMAKE_ARGS -DCMAKE_C_FLAGS=${MKLDNN_CFLAG} + CMAKE_ARGS -DCMAKE_CXX_FLAGS=${MKLDNN_CXXFLAG} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${MKLDNN_INSTALL_DIR} -DMKLROOT:PATH=${MKLDNN_MKLROOT} ) diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index 74f3279831357c21038df133df0f5a432a6dfd20..20dbc32a738d982df2d3f035206279c82c8de264 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -27,8 +27,8 @@ ENDIF() INCLUDE(ExternalProject) SET(MKLML_PROJECT "extern_mklml") -SET(MKLML_VER "mklml_lnx_2018.0.20170720") -SET(MKLML_URL "https://github.com/01org/mkl-dnn/releases/download/v0.10/${MKLML_VER}.tgz") +SET(MKLML_VER "mklml_lnx_2018.0.1.20171007") +SET(MKLML_URL "https://github.com/01org/mkl-dnn/releases/download/v0.11/${MKLML_VER}.tgz") SET(MKLML_SOURCE_DIR "${THIRD_PARTY_PATH}/mklml") SET(MKLML_DOWNLOAD_DIR "${MKLML_SOURCE_DIR}/src/${MKLML_PROJECT}") SET(MKLML_DST_DIR "mklml") diff --git a/cmake/external/nccl.cmake b/cmake/external/nccl.cmake index 57d2c0a352507afd01d1cbf2c7b23c00ff7ad81b..fc43766efafc3d3e16f2906ce7f9a3d692c8e4ff 100644 --- a/cmake/external/nccl.cmake +++ b/cmake/external/nccl.cmake @@ -1,3 +1,21 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(NOT WITH_GPU) + return() +endif() + include(ExternalProject) set(NCCL_SOURCE_DIR ${THIRD_PARTY_PATH}/nccl) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 143b57a954e4e6b2bf273535ebdf0fa8e3dab768..05d83ad58ef8485d36829e7aeede79f625cfdc43 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -86,7 +86,7 @@ IF(NOT ${CBLAS_FOUND}) UPDATE_COMMAND "" CONFIGURE_COMMAND "" ) - + SET(CBLAS_PROVIDER openblas) IF(WITH_C_API) INSTALL(DIRECTORY ${CBLAS_INC_DIR} DESTINATION third_party/openblas) # Because libopenblas.a is a symbolic link of another library, thus need to @@ -115,7 +115,7 @@ INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) # linear algebra libraries for cc_library(xxx SRCS xxx.c DEPS cblas) SET(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/cblas_dummy.c) FILE(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") -IF(${CBLAS_PROVIDER} MATCHES MKL) +IF("${CBLAS_PROVIDER}" STREQUAL "MKLML") ADD_LIBRARY(cblas SHARED ${dummyfile}) ELSE() ADD_LIBRARY(cblas STATIC ${dummyfile}) diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake index 9391c285c7544669a5b1a078b7473d7a656c1bb4..4e87dc49d8956d1fa6dec777efc5a63c6b0f79a5 100644 --- a/cmake/external/pybind11.cmake +++ b/cmake/external/pybind11.cmake @@ -1,8 +1,26 @@ -INCLUDE(ExternalProject) +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -SET(PYBIND_SOURCE_DIR ${THIRD_PARTY_PATH}/pybind) +if(NOT WITH_PYTHON) + return() +endif() + +include(ExternalProject) -INCLUDE_DIRECTORIES(${PYBIND_SOURCE_DIR}/src/extern_pybind/include) +set(PYBIND_SOURCE_DIR ${THIRD_PARTY_PATH}/pybind) + +include_directories(${PYBIND_SOURCE_DIR}/src/extern_pybind/include) ExternalProject_Add( extern_pybind @@ -17,14 +35,12 @@ ExternalProject_Add( TEST_COMMAND "" ) -if (${CMAKE_VERSION} VERSION_LESS "3.3.0") +if(${CMAKE_VERSION} VERSION_LESS "3.3.0") set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/pybind_dummy.c) - file(WRITE ${dummyfile} "const char * dummy_any = \"${dummyfile}\";") + file(WRITE ${dummyfile} "const char * dummy_pybind = \"${dummyfile}\";") add_library(pybind STATIC ${dummyfile}) else() add_library(pybind INTERFACE) endif() add_dependencies(pybind extern_pybind) - -LIST(APPEND external_project_dependencies pybind) diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake index ce088ae7eaa3355f2f9761e8c421da0d7ef89fa7..9db457c7b2d61228e5d5af6827c4cda11a20a463 100644 --- a/cmake/external/swig.cmake +++ b/cmake/external/swig.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index e2c9fe56f335ae5b627b4d8d4bb17e4a2a466677..a98e069b7cd1654ddd5868560d0905eab6d9c692 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/cmake/generic.cmake b/cmake/generic.cmake index c311783aa3187678c31c27ddbbd074790ca444f3..b9c1dde97bc444d793d67ff622fd6b13c6435a9a 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -93,7 +93,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(NOT APPLE AND NOT ANDROID) find_package(Threads REQUIRED) link_libraries(${CMAKE_THREAD_LIBS_INIT}) - set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -ldl -lrt") + set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt") endif(NOT APPLE AND NOT ANDROID) function(merge_static_libs TARGET_NAME) diff --git a/cmake/simd.cmake b/cmake/simd.cmake index 46035a908b588861607a25d3a21cf34b7b6fd4b8..53c2de332ea74b06d1bd6e5bb119cad6af27ed01 100644 --- a/cmake/simd.cmake +++ b/cmake/simd.cmake @@ -1,27 +1,28 @@ # This file is use to check all support level of AVX on your machine # so that PaddlePaddle can unleash the vectorization power of muticore. -INCLUDE(CheckCXXSourceRuns) -INCLUDE(CheckCXXSourceCompiles) +include(CheckCXXSourceRuns) +include(CheckCXXSourceCompiles) -IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") +if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(MMX_FLAG "-mmmx") set(SSE2_FLAG "-msse2") set(SSE3_FLAG "-msse3") - SET(AVX_FLAG "-mavx") - SET(AVX2_FLAG "-mavx2") -ELSEIF(MSVC) + set(AVX_FLAG "-mavx") + set(AVX2_FLAG "-mavx2") +elseif(MSVC) set(MMX_FLAG "/arch:MMX") set(SSE2_FLAG "/arch:SSE2") set(SSE3_FLAG "/arch:SSE3") SET(AVX_FLAG "/arch:AVX") SET(AVX2_FLAG "/arch:AVX2") -ENDIF() +endif() set(CMAKE_REQUIRED_FLAGS_RETAINED ${CMAKE_REQUIRED_FLAGS}) # Check MMX set(CMAKE_REQUIRED_FLAGS ${MMX_FLAG}) +set(MMX_FOUND_EXITCODE 1 CACHE STRING "Result from TRY_RUN" FORCE) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -32,6 +33,7 @@ int main() # Check SSE2 set(CMAKE_REQUIRED_FLAGS ${SSE2_FLAG}) +set(SSE2_FOUND_EXITCODE 1 CACHE STRING "Result from TRY_RUN" FORCE) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -42,6 +44,7 @@ int main() # Check SSE3 set(CMAKE_REQUIRED_FLAGS ${SSE3_FLAG}) +set(SSE3_FOUND_EXITCODE 1 CACHE STRING "Result from TRY_RUN" FORCE) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -55,6 +58,7 @@ int main() # Check AVX set(CMAKE_REQUIRED_FLAGS ${AVX_FLAG}) +set(AVX_FOUND_EXITCODE 1 CACHE STRING "Result from TRY_RUN" FORCE) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -67,6 +71,7 @@ int main() # Check AVX 2 set(CMAKE_REQUIRED_FLAGS ${AVX2_FLAG}) +set(AVX2_FOUND_EXITCODE 1 CACHE STRING "Result from TRY_RUN" FORCE) CHECK_CXX_SOURCE_RUNS(" #include int main() diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index d4e9d53e5c0955912a594fe8cd9cd41a4080a2d2..203506d7ab84e5a5be2232b077eac2d433a99766 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -82,6 +82,11 @@ maxout .. autoclass:: paddle.v2.layer.maxout :noindex: +roi_pool +-------- +.. autoclass:: paddle.v2.layer.roi_pool + :noindex: + Norm Layer ========== diff --git a/doc/api/v2/data.rst b/doc/api/v2/data.rst index fef87c4fbdb452771ecdb361c6eeae5b32bcee14..b56c7332cc284649c7e04328e51a7faa78593a39 100644 --- a/doc/api/v2/data.rst +++ b/doc/api/v2/data.rst @@ -2,112 +2,9 @@ Data Reader Interface and DataSets ================================== +.. toctree:: + :maxdepth: 1 -DataTypes -========= - -.. automodule:: paddle.v2.data_type - :members: - :noindex: - -DataFeeder -========== - -.. automodule:: paddle.v2.data_feeder - :members: - :noindex: - -Reader -====== - -.. automodule:: paddle.v2.reader - :members: - :noindex: - -.. automodule:: paddle.v2.reader.creator - :members: - :noindex: - -minibatch -========= - -.. automodule:: paddle.v2.minibatch - :members: - :noindex: - -Dataset -======= - -.. automodule:: paddle.v2.dataset - :members: - :noindex: - -mnist -+++++ - -.. automodule:: paddle.v2.dataset.mnist - :members: - :noindex: - -cifar -+++++ - -.. automodule:: paddle.v2.dataset.cifar - :members: - :noindex: - -conll05 -+++++++ - -.. automodule:: paddle.v2.dataset.conll05 - :members: get_dict,get_embedding,test - :noindex: - -imdb -++++ - -.. automodule:: paddle.v2.dataset.imdb - :members: - :noindex: - -imikolov -++++++++ - -.. automodule:: paddle.v2.dataset.imikolov - :members: - :noindex: - -movielens -+++++++++ - -.. automodule:: paddle.v2.dataset.movielens - :members: - :noindex: - -.. autoclass:: paddle.v2.dataset.movielens.MovieInfo - :noindex: - -.. autoclass:: paddle.v2.dataset.movielens.UserInfo - :noindex: - -sentiment -+++++++++ - -.. automodule:: paddle.v2.dataset.sentiment - :members: - :noindex: - -uci_housing -+++++++++++ - -.. automodule:: paddle.v2.dataset.uci_housing - :members: - :noindex: - -wmt14 -+++++ - -.. automodule:: paddle.v2.dataset.wmt14 - :members: - :noindex: - + data/data_reader.rst + data/image.rst + data/dataset.rst diff --git a/doc/api/v2/data/data_reader.rst b/doc/api/v2/data/data_reader.rst new file mode 100644 index 0000000000000000000000000000000000000000..2ccfec9c284877a7576e9751526b169a4ac78d8e --- /dev/null +++ b/doc/api/v2/data/data_reader.rst @@ -0,0 +1,36 @@ +===================== +Data Reader Interface +===================== + + +DataTypes +========= + +.. automodule:: paddle.v2.data_type + :members: + :noindex: + +DataFeeder +========== + +.. automodule:: paddle.v2.data_feeder + :members: + :noindex: + +Reader +====== + +.. automodule:: paddle.v2.reader + :members: + :noindex: + +.. automodule:: paddle.v2.reader.creator + :members: + :noindex: + +minibatch +========= + +.. automodule:: paddle.v2.minibatch + :members: + :noindex: diff --git a/doc/api/v2/data/dataset.rst b/doc/api/v2/data/dataset.rst new file mode 100644 index 0000000000000000000000000000000000000000..6a8ecc5bb1d855e0ded3719943ab3adb810de365 --- /dev/null +++ b/doc/api/v2/data/dataset.rst @@ -0,0 +1,75 @@ +Dataset +======= + +.. automodule:: paddle.v2.dataset + :members: + :noindex: + +mnist ++++++ + +.. automodule:: paddle.v2.dataset.mnist + :members: + :noindex: + +cifar ++++++ + +.. automodule:: paddle.v2.dataset.cifar + :members: + :noindex: + +conll05 ++++++++ + +.. automodule:: paddle.v2.dataset.conll05 + :members: get_dict,get_embedding,test + :noindex: + +imdb +++++ + +.. automodule:: paddle.v2.dataset.imdb + :members: + :noindex: + +imikolov +++++++++ + +.. automodule:: paddle.v2.dataset.imikolov + :members: + :noindex: + +movielens ++++++++++ + +.. automodule:: paddle.v2.dataset.movielens + :members: + :noindex: + +.. autoclass:: paddle.v2.dataset.movielens.MovieInfo + :noindex: + +.. autoclass:: paddle.v2.dataset.movielens.UserInfo + :noindex: + +sentiment ++++++++++ + +.. automodule:: paddle.v2.dataset.sentiment + :members: + :noindex: + +uci_housing ++++++++++++ + +.. automodule:: paddle.v2.dataset.uci_housing + :members: + :noindex: + +wmt14 ++++++ + +.. automodule:: paddle.v2.dataset.wmt14 + :members: + :noindex: diff --git a/doc/api/v2/data/image.rst b/doc/api/v2/data/image.rst new file mode 100644 index 0000000000000000000000000000000000000000..97651ffa6be56cf3ecaca2caca38a353fa5c1f49 --- /dev/null +++ b/doc/api/v2/data/image.rst @@ -0,0 +1,5 @@ +Image Interface +=============== + +.. automodule:: paddle.v2.image + :members: diff --git a/doc/design/float16.md b/doc/design/float16.md new file mode 100644 index 0000000000000000000000000000000000000000..078801ba2ed969d26dd31d5ec4ed268686cf7016 --- /dev/null +++ b/doc/design/float16.md @@ -0,0 +1,60 @@ +# Design Doc: float16 + +## Why float16 +Half precision (float16) is a binary floating-point format that occupies 16 bits in memory. float16 is half the size of traditional 32-bit single precision format (float) and has lower precision and smaller range. + +When high precision computation is not required, using float16 data type could potentially + +- reduce storage space, memory bandwidth, and power usages; +- increase the chance of data fitting into a smaller cache of lower latency; +- provide arithmetic speed up if supported by hardware. + +## Survey of current float16 support +A brief survey of float16 support on different compilers, hardwares, and libraries can be found below. Interested readers can refer to [link1](https://github.com/PaddlePaddle/Paddle/issues/4853) and [link2](https://github.com/Xreki/Xreki.github.io/blob/master/multi_data_types_in_dl_framework/ppt/float16_and_quantized_type.md) for more info. + +The goal of float16 is to serve as a key for the executor to find and run the correct version of compute method specialized for float16 in operator kernel. It should be compatible with various natively supported float16 implementations including `__half` for cuda, `float16_t` for ARM, and `Eigen::half` for Eigen to make writing customized float16 kernels easier. + +### Compiler +- nvcc supports `__half` data type after CUDA 7.5. +- `__fp16` or `float16_t` is supported as storage type for gcc >= 6.1 and clang >= 3.4. +- `__fp16` or `float16_t` is supported as arithmetic type for gcc >= 7.1 and clang >= 3.9. + +### Hardware +- `__half` is supported on GPU with compute capability >= 5.3. +- `__fp16` is supported as storage type for ARMv7-A, ARMv8-A, and above. +- `__fp16` is supported as arithmetic type after ARMv8.2-A (currently, the only microarchitecture implementing ARMv8.2-A is ARM Cortex-A75, which is announced in May 2017. There seems to be no application processors currently available on market that adopts this architecture. It is reported that Qualcomm Snapdragon 845 uses Cortex-A75 design and will be available in mobile devices in early 2018). + +### Libraries +- [Eigen](https://github.com/RLovelett/eigen) >= 3.3 supports float16 calculation on both GPU and CPU using the `Eigen::half` class. It is mostly useful for Nvidia GPUs because of the overloaded arithmetic operators using cuda intrinsics. It falls back to using software emulation on CPU for calculation and there is no special treatment to ARM processors. +- [ARM compute library](https://github.com/ARM-software/ComputeLibrary) >= 17.02.01 supports NEON FP16 kernels (requires ARMv8.2-A CPU). + + +## Implementation +The float16 class holds a 16-bit `uint16_t` data internally. +``` +struct float16 { + uint16_t x; +}; +``` + +float16 supports the following features: + - constructors / assignment operators that take input from primitive data types including bool, integers of various length, float, and double. + - constructors / assignment operators that take input from `__half` on cuda, `float16_t` on ARM, and `Eigen::half` on Eigen. + - conversion operators to primitive data types and half precision data types on cuda, ARM and Eigen. + - overloaded arithmetic operators for cuda, arm, and non-arm cpu, respectively. These operators will take advantage of the cuda and ARM intrinsics on the corresponding hardware. + +To support the above features, two fundamental conversion functions are provided: +``` +float16 float_to_half_rn(float f); // convert to half precision in round-to-nearest-even mode +float half_to_float(float16 h); +``` +which provides one-to-one conversion between float32 and float16. These twos functions will do different conversion routines based on the current hardware. CUDA/ARM instrinsics will be used when the corresonding hardware is available. If the hardware or compiler level does not support float32 to float16 conversion, software emulation will be performed to do the conversion. + +## To do +After float16 class is available, some of the future items are below: + +- Update pybind/tensor_py.h to bind c++ float16 with numpy float16. + +- Modify `GetKernelType()` method in `framework/operator.h` to make it compatible with float16. + +- Create a type-casting operator that can convert the data type in tensor between float16 and other types. diff --git a/doc/design/images/asgd.gif b/doc/design/images/asgd.gif new file mode 100644 index 0000000000000000000000000000000000000000..4a0da7bf6df9326a2aab1638b77c5455c18b8c4e Binary files /dev/null and b/doc/design/images/asgd.gif differ diff --git a/doc/design/images/theta_star.gif b/doc/design/images/theta_star.gif new file mode 100644 index 0000000000000000000000000000000000000000..dd24d33e124396be3fc410c9b12f33148f64efe2 Binary files /dev/null and b/doc/design/images/theta_star.gif differ diff --git a/doc/design/mkldnn/README.MD b/doc/design/mkldnn/README.MD index fe8da907d9d45a2164031430ac5b7a3d5523967a..16236763a73770f3fe5eadf67645765d0456f875 100644 --- a/doc/design/mkldnn/README.MD +++ b/doc/design/mkldnn/README.MD @@ -15,6 +15,7 @@ - [CMake](#cmake) - [Layers](#layers) - [Activations](#activations) + - [Weights](#weights) - [Unit Tests](#unit-tests) - [Protobuf Messages](#protobuf-messages) - [Python API](#python-api) @@ -45,17 +46,23 @@ Figure 1. PaddlePaddle on IA. ### Layers 所有MKL-DNN相关的C++ layers，都会按照PaddlePaddle的目录结构存放在 -`paddle/gserver/layers`中，并且文件名都会一以*Mkldnn*开头。 +`paddle/gserver/layers`中，并且文件名都会一以*MKLDNN*开头。 -所有MKL-DNN的layers都会继承于一个叫做`MkldnnLayer`的父类，该父类继承于PaddlePaddle的基类`Layer`。 +所有MKL-DNN的layers都会继承于一个叫做`MKLDNNLayer`的父类，该父类继承于PaddlePaddle的基类`Layer`。 + +在`MKLDNNLayer`中会提供一些必要的接口和函数，并且会写好`forward`和`backward`的基本逻辑。部分函数定义为纯虚函数，子类只需要实现这些函数即可。 ### Activations -由于在PaddlePaddle中，激活函数是独立于layer概念的，所以会在`paddle/gserver/activations`目录下添加一个`MkldnnActivation.h`文件定义一些用于MKL-DNN的接口，实现方法还是会在`ActivationFunction.cpp`文件。 +由于在PaddlePaddle中，激活函数是独立于layer概念的，所以会在`paddle/gserver/activations`目录下添加`MKLDNNActivation.h`和`MKLDNNActivation.cpp`文件用于定义和使用MKL-DNN的接口。 -### Unit Tests -会在`paddle/gserver/test`目录下添加`test_Mkldnn.cpp`和`MkldnnTester.*`用于MKL-DNN的测试。 +### Weights +由于有些layer是含有参数的，我们会尽量让MKL-DNN的参数与PaddlePaddle中`parameter`共享一块内存。 +同时，由于MKL-DNN在训练时使用的参数layout可能与PaddlePaddle默认的`nchw`不一致，我们会在网络训练的开始和结束时分别转换这个layout，使得最终保存的参数格式与PaddlePaddle一致。 -Activation的测试，计划在PaddlePaddle原有的测试文件上直接添加新的测试type。 +### Unit Tests +会在`paddle/gserver/test`目录下添加`test_MKLDNN.cpp`和`MKLDNNTester.*`用于MKL-DNN的测试。 +测试分为每个layer(或activation)的单元测试和简单网络的整体测试。 +每个测试会对比PaddlePaddle中CPU算出的结果与MKL-DNN的结果，小于某个比较小的阈值认为通过。 ### Protobuf Messages 根据具体layer的需求可能会在`proto/ModelConfig.proto`里面添加必要的选项。 @@ -82,7 +89,7 @@ if use_mkldnn 会在`v1_api_demo`目录下添加一个`mkldnn`的文件夹，里面放入一些用于MKL-DNN测试的demo脚本。 ### Benchmarking -会考虑添加部分逻辑在`benchmark/paddle/image/run.sh`，添加使用MKL-DNN的测试。 +会添加`benchmark/paddle/image/run_mkldnn.sh`，用于测试使用MKL-DNN之后的性能。 ### Others 1. 如果在使用MKL-DNN的情况下，会把CPU的Buffer对齐为64。 @@ -94,14 +101,16 @@ if use_mkldnn 我们总结出一些特别需要注意的点： -1. 使用**deviceId_**。为了尽可能少的在父类Layer中添加变量或者函数，我们决定使用已有的`deviceId_`变量来区分layer的属性，定义`-2`为`MkldnnLayer`特有的设备ID。 +1. 使用**deviceId_**。为了尽可能少的在父类Layer中添加变量或者函数，我们决定使用已有的`deviceId_`变量来区分layer的属性，定义`-2`为`MKLDNNLayer`特有的设备ID。 2. 重写父类Layer的**init**函数，修改`deviceId_`为`-2`，代表这个layer是用于跑在MKL-DNN的环境下。 -3. 创建`MkldnnMatrix`，用于管理MKL-DNN会用到的相关memory函数、接口以及会用的到格式信息。 -4. 创建`MkldnnBase`，定义一些除了layer和memory相关的类和函数。包括MKL-DNN会用到`MkldnnStream`和`CpuEngine`，和未来可能还会用到`FPGAEngine`等。 -5. 在**Argument**里添加两个`MkldnnMatrixPtr`，取名为`mkldnnValue`和`mkldnnGrad`，用于存放`MkldnnLayer`会用到的memory buffer。 并且添加函数cvt(会修改为一个更加合适的函数名)，用于处理"CPU device"和"MKL-DNN device"之间memory的相互转化。 -6. 在父类`Layer`中的`getOutput`函数中添加一段逻辑，用于判断`deviceId`，并针对device在MKL-DNN和CPU之间不统一的情况，做一个前期转换。 也就是调用`Argument`的cvt函数把output统一到需要的device上。 -7. 在原来的`FLAGS`中添加一个`use_mkldnn`的flag，用于选择是否使用MKL-DNN的相关功能。 -8. 关于MKLDNN参数的保存。由于MKLDNN参数的格式与PaddlePaddle原有的格式存在不一样的情况，所以需要在保存参数时同时保存该格式信息。目前准备扩展[Header](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/parameter/Parameter.h#L247)里面的`int32_t version`。这个值不管是在v1还是在v2里面，一直保存的是0，所以可以充分利用这个信息，定义一个枚举处理所有MKLDNN的参数格式，从而`MKLDNNLayer`就可以从输入的参数中获取需要的格式信息。 +3. 创建`MKLDNNMatrix`，同时继承`CpuMatrix`和`mkldnn::memory`。用于管理MKL-DNN会用到的相关memory函数、接口以及会用的到格式信息。 +4. 创建`MKLDNNBase`，定义一些除了layer和memory相关的类和函数。包括MKL-DNN会用到`MKLDNNStream`和`CPUEngine`，和未来可能还会用到`FPGAEngine`等。 +5. 每个`MKLDNNlayer`都会有`inVal_`,`inGrad_`,`outVal_`和`outGrad_`，分别代表input value， input gradient，output value和output gradient。他们会存放MKL-DNN用到的internal memory。同时还会定义以*ext*开头的`MKLDNNMatrix`(表示external的memory)，主要是在格式与PaddlePaddle默认的`nchw`格式不匹配时，用于转换内存的工作。必要的转换函数也会在`MKLDNNLayer`中提前定义好，每个子类只需要调用定义好的reset buffer函数即可。 +6. 每个`MKLDNNlayer`的resetbuffer相关的函数（包括reset input、output的Value和grad），他们会根据输入参数reset internal和external的memory，当然这两者也可以相等，即表示不需要转换。只需要把握一个原则，每个`MKLDNNlayer`的子类，只需要使用internal的memory就可以了，所有external的转换工作在父类的reset函数中都提前准备好了。 +7. 一般来说，external的memory会尽量与PaddlePaddle中的`value`和`grad`共享内存。同时每个`MKLDNNLayer`中的external output value和gradient(也就是`extOutVal_`和`extOutGrad_`)必须分别与`output_.value`和`output_.grad`共享内存，因为PaddlePaddle的activation会直接使用`output_.value`和`output_.grad`。如果不需要external的buffer用于转换，那么internal的buffer也会与他们共享内存。 +8. 如果MKL-DNN layer的后面接有cpu device，那么就会使`output_.value`与`extOutVal_`共享内存，同时数据格式就是`nchw`，这样下一个cpu device就能拿到正确的数据。在有cpu device的时候，external的memory的格式始终是`nchw`或者`nc`。 +9. 由于MKL-DNN的输出操作都是覆盖data的，不是在原来的数据上累加，所以当网络出现分支时，在`backward`时会需要merge不同layer的梯度。`MKLDNNlayer`中会实现merge的方法，此时每个小分支的input gradient会先临时保存在一个`MKLDNNMatrix`中，由分支处的layer负责求和，并把结果放到这个layer的`output_.grad`中。所以整体上，每个子类并不会需要关心分支的事情，也是在父类都实现好了。 +10. 在原来的`FLAGS`中添加一个`use_mkldnn`的flag，用于选择是否使用MKL-DNN的相关功能。 ## References diff --git a/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg b/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8b0d90f7b9d8184b314b0ee4e521f53eb5f1b455 Binary files /dev/null and b/doc/design/ops/images/LOD-and-shape-changes-during-decoding.jpg differ diff --git a/doc/design/ops/sequence_decoder.md b/doc/design/ops/sequence_decoder.md new file mode 100644 index 0000000000000000000000000000000000000000..9007aae7a8355ed06c6720a921351f81b859c1fe --- /dev/null +++ b/doc/design/ops/sequence_decoder.md @@ -0,0 +1,245 @@ +# Design: Sequence Decoder Generating LoDTensors +In tasks such as machine translation and image to text, +a [sequence decoder](https://github.com/PaddlePaddle/book/blob/develop/08.machine_translation/README.md) is necessary to generate sequences. + +This documentation describes how to implement the sequence decoder as an operator. + +## Beam Search based Decoder +The [beam search algorithm](https://en.wikipedia.org/wiki/Beam_search) is necessary when generating sequences, +it is a heuristic search algorithm that explores the paths by expanding the most promising node in a limited set. + +In the old version of PaddlePaddle, a C++ class `RecurrentGradientMachine` implements the general sequence decoder based on beam search, +due to the complexity, the implementation relays on a lot of special data structures, +quite trivial and hard to be customized by users. + +There are a lot of heuristic tricks in the sequence generation tasks, +so the flexibility of sequence decoder is very important to users. + +During PaddlePaddle's refactoring work, +some new concept is proposed such as [LoDTensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/lod_tensor.md) and [TensorArray](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/tensor_array.md) that can better support sequence usage, +and they can help to make the implementation of beam search based sequence decoder **more transparent and modular** . + +For example, the RNN sates, candidates IDs and probabilities of beam search can be represented as `LoDTensors`; +the selected candidate's IDs in each time step can be stored in a `TensorArray`, and `Packed` to the sentences translated. + +## Changing LoD's absolute offset to relative offsets +The current `LoDTensor` is designed to store levels of variable-length sequences, +it stores several arrays of integers each represents a level. + +The integers in each level represents the begin and end (not inclusive) offset of a sequence **in the underlying tensor**, +let's call this format the **absolute-offset LoD** for clear. + +The relative-offset LoD can fast retrieve any sequence but fails to represent empty sequences, for example, a two-level LoD is as follows +```python +[[0, 3, 9] + [0, 2, 3, 3, 3, 9]] +``` +The first level tells that there are two sequences: +- the first's offset is `[0, 3)` +- the second's offset is `[3, 9)` + +while on the second level, there are several empty sequences that both begin and end at `3`. +It is impossible to tell how many empty second-level sequences exist in the first-level sequences. + +There are many scenarios that relay on empty sequence representation, +such as machine translation or image to text, one instance has no translations or the empty candidate set for a prefix. + +So let's introduce another format of LoD, +it stores **the offsets of the lower level sequences** and is called **relative-offset** LoD. + +For example, to represent the same sequences of the above data + +```python +[[0, 3, 6] + [0, 2, 3, 3, 3, 9]] +``` + +the first level represents that there are two sequences, +their offsets in the second-level LoD is `[0, 3)` and `[3, 5)`. + +The second level is the same with the relative offset example because the lower level is a tensor. +It is easy to find out the second sequence in the first-level LoD has two empty sequences. + +The following demos are based on relative-offset LoD. + +## Usage in a simple machine translation model +Let's start from a simple machine translation model that is simplified from [machine translation chapter](https://github.com/PaddlePaddle/book/tree/develop/08.machine_translation) to draw a simple blueprint of what a sequence decoder can do and how to use it. + +The model has an encoder that learns the semantic vector from a sequence, +and a decoder which uses the sequence decoder to generate new sentences. + +**Encoder** +```python +import paddle as pd + +dict_size = 8000 +source_dict_size = dict_size +target_dict_size = dict_size +word_vector_dim = 128 +encoder_dim = 128 +decoder_dim = 128 +beam_size = 5 +max_length = 120 + +# encoder +src_word_id = pd.data( + name='source_language_word', + type=pd.data.integer_value_sequence(source_dict_dim)) +src_embedding = pd.embedding(size=source_dict_size, size=word_vector_dim) + +src_word_vec = pd.lookup(src_embedding, src_word_id) + +encoder_out_seq = pd.gru(input=src_word_vec, size=encoder_dim) + +encoder_ctx = pd.last_seq(encoder_out_seq) +# encoder_ctx_proj is the learned semantic vector +encoder_ctx_proj = pd.fc( + encoder_ctx, size=decoder_dim, act=pd.activation.Tanh(), bias=None) +``` + +**Decoder** + +```python +def generate(): + decoder = pd.while_loop() + with decoder.step(): + decoder_mem = decoder.memory(init=encoder_ctx) # mark the memory + generated_ids = decoder.memory() # TODO init to batch_size s + generated_scores = decoder.memory() # TODO init to batch_size 1s or 0s + + target_word = pd.lookup(trg_embedding, gendrated_ids) + # expand encoder_ctx's batch to fit target_word's lod + # for example + # decoder_mem.lod is + # [[0 1 3], + # [0 1 3 6]] + # its tensor content is [a1 a2 a3 a4 a5] + # which means there are 2 sentences to translate + # - the first sentence has 1 translation prefixes, the offsets are [0, 1) + # - the second sentence has 2 translation prefixes, the offsets are [1, 3) and [3, 6) + # the target_word.lod is + # [[0, 1, 6] + # [0, 2, 4, 7, 9 12]] + # which means 2 sentences to translate, each has 1 and 5 prefixes + # the first prefix has 2 candidates + # the following has 2, 3, 2, 3 candidates + # the encoder_ctx_expanded's content will be + # [a1 a1 a2 a2 a3 a3 a3 a4 a4 a5 a5 a5] + encoder_ctx_expanded = pd.lod_expand(encoder_ctx, target_word) + decoder_input = pd.fc( + act=pd.activation.Linear(), + input=[target_word, encoder_ctx], + size=3 * decoder_dim) + gru_out, cur_mem = pd.gru_step( + decoder_input, mem=decoder_mem, size=decoder_dim) + scores = pd.fc( + gru_out, + size=trg_dic_size, + bias=None, + act=pd.activation.Softmax()) + # K is an config + topk_scores, topk_ids = pd.top_k(scores, K) + topk_generated_scores = pd.add_scalar(topk_scores, generated_scores) + + selected_ids, selected_generation_scores = decoder.beam_search( + topk_ids, topk_generated_scores) + + # update the states + decoder_mem.update(cur_mem) # tells how to update state + generated_ids.update(selected_ids) + generated_scores.update(selected_generation_scores) + + decoder.output(selected_ids) + decoder.output(selected_generation_scores) + +translation_ids, translation_scores = decoder() +``` +The `decoder.beam_search` is a operator that given the candidates and the scores of translations including the candidates, +return the result of the beam search algorithm. + +In this way, users can customize anything on the inputs or outputs of beam search, for example, two ways to prune some translation prefixes + +1. meke the correspondind elements in `topk_generated_scores` zero or some small values, beam_search will discard this candidate. +2. remove some specific candidate in `selected_ids` +3. get the final `translation_ids`, remove the translation sequence in it. + +The implementation of sequence decoder can reuse the C++ class [RNNAlgorithm](https://github.com/Superjom/Paddle/blob/68cac3c0f8451fe62a4cdf156747d6dc0ee000b3/paddle/operators/dynamic_recurrent_op.h#L30), +so the python syntax is quite similar to a [RNN](https://github.com/Superjom/Paddle/blob/68cac3c0f8451fe62a4cdf156747d6dc0ee000b3/doc/design/block.md#blocks-with-for-and-rnnop). + +Both of them are two-level `LoDTensors` + +- the first level represents `batch_size` of (source) sentences; +- the second level represents the candidate ID sets for translation prefix. + +for example, 3 source sentences to translate, and has 2, 3, 1 candidates. + +Unlike an RNN, in sequence decoder, the previous state and the current state have different LoD and shape, +a `lod_expand` operator is used to expand the LoD of the previous state to fit the current state. + +For example, the previous state + +* LoD is `[0, 1, 3][0, 2, 5, 6]` +* content of tensor is `a1 a2 b1 b2 b3 c1` + +the current state stored in `encoder_ctx_expanded` + +* LoD is `[0, 2, 7][0 3 5 8 9 11 11]` +* the content is + - a1 a1 a1 (a1 has 3 candidates, so the state should be copied 3 times for each candidates) + - a2 a2 + - b1 b1 b1 + - b2 + - b3 b3 + - None (c1 has 0 candidates, so c1 is dropped) + +Benefit from the relative offset LoD, empty candidate set can be represented naturally. + +the status in each time step can be stored in `TensorArray`, and `Pack`ed to a final LoDTensor, the corresponding syntax is + +```python +decoder.output(selected_ids) +decoder.output(selected_generation_scores) +``` + +the `selected_ids` is the candidate ids for the prefixes, +it will be `Packed` by `TensorArray` to a two-level `LoDTensor`, +the first level represents the source sequences, +the second level represents generated sequences. + +Pack the `selected_scores` will get a `LoDTensor` that stores scores of each candidate of translations. + +Pack the `selected_generation_scores` will get a `LoDTensor`, and each tail is the probability of the translation. + +## LoD and shape changes during decoding +

+ +

+ +According the image above, the only phrase to change LoD is beam search. + +## Beam search design +The beam search algorthm will be implemented as one method of the sequence decoder, it has 3 inputs + +1. `topk_ids`, top K candidate ids for each prefix. +2. `topk_scores`, the corresponding scores for `topk_ids` +3. `generated_scores`, the score of the prefixes. + +All of the are LoDTensors, so that the sequence affilication is clear. +Beam search will keep a beam for each prefix and select a smaller candidate set for each prefix. + +It will return three variables + +1. `selected_ids`, the final candidate beam search function selected for the next step. +2. `selected_scores`, the scores for the candidates. +3. `generated_scores`, the updated scores for each prefixes (with the new candidates appended). + +## Introducing the LoD-based `Pack` and `Unpack` methods in `TensorArray` +The `selected_ids`, `selected_scores` and `generated_scores` are LoDTensors, +and they exist in each time step, +so it is natural to store them in arrays. + +Currently, PaddlePaddle has a module called `TensorArray` which can store an array of tensors, +the results of beam search are better to store in a `TensorArray`. + +The `Pack` and `UnPack` in `TensorArray` are used to package tensors in the array to a `LoDTensor` or split the `LoDTensor` to an array of tensors. +It needs some extensions to support pack or unpack an array of `LoDTensors`. diff --git a/doc/design/parameter_average.md b/doc/design/parameter_average.md new file mode 100644 index 0000000000000000000000000000000000000000..2c4edee9fe31d502ea62b9fe5c8757c0a4c5e79f --- /dev/null +++ b/doc/design/parameter_average.md @@ -0,0 +1,72 @@ +# Averaging Parameter in PaddlePaddle + +## Why Averaging +In a large scale machine learning setup where the size of the training data is huge, it could take us a large number of iterations over the training data before we can achieve the optimal values of parameters of our model. Looking at the problem setup, it is desirable if we can obtain the optimal values of parameters by going through the data in as few passes as we can. + +Polyak and Juditsky (1992) showed that the test performance of simple average of parameters obtained by Stochastic Gradient Descent (SGD) is as good as that of parameter values that are obtained by training the model over and over again, over the training dataset. + +Hence, to accelerate the speed of Stochastic Gradient Descent, Averaged Stochastic Gradient Descent (ASGD) was proposed in Polyak and Juditsky (1992). For ASGD, the running average of parameters obtained by SGD, is used as the estimator for
. The averaging is done as follows: + +
+ +We propose averaging for any optimizer similar to how ASGD performs it, as mentioned above. + +### How to perform Parameter Averaging in PaddlePaddle + +Parameter Averaging in PaddlePaddle works in the following way during training : +1. It will take in an instance of a normal optimizer as an input, e.g. RMSPropOptimizer +2. The optimizer itself is responsible for updating the parameters. +3. The ParameterAverageOptimizer maintains a separate copy of the parameters for itself: + 1. In concept, the values of this copy are the average of the values of the parameters in the most recent N batches. + 2. However, saving all the N instances of the parameters in memory is not feasible. + 3. Therefore, an approximation algorithm is used. + +Hence, overall we have have two copies of the parameters: one for the optimizer itself, and one for the ParameterAverageOptimizer. The former should be used in back propagation, while the latter should be used during testing and should be saved. + +During the testing/ saving the model phase, we perform the following steps: +1. Perform the delayed operations. +2. Save current values of the parameters to a temporary variable. +3. Replace the values of the parameters with the averaged values. +4. Perform testing and/or save the parameters. +5. Restore the values of the parameters once done. + +### How to implement Averaging of Parameter in PaddlePaddle + +We can add the ParameterAverageOptimizer op to the graph through Python API. Using this approach, we manually add this op to the graph and direct the output of the optimizer op to this op during training. + + **Advantages**: + - Allows for greater flexibility to the users of PaddlePaddle. Using this approach, the users can plug different optimizers into ParameterAverageOptimizer by passing in the optimizer to the op. + - Makes it easy for the users to customize and extend the framework. + + **Disadvantages**: + - Implementation requires re-writing the averaging methodology in Python. + +### Low-Level implementation + +In the new design, we propose to create a new operation for averaging parameter updates (ParameterAverageOptimizer). For now, we can add an op that takes in the following as input: +- the optimizer +- the window_size to keep the updates + +The ParameterAverageOptimizer op can be like any other operator with its own CPU/GPU implementation either using Eigen or separate CPU and GPU kernels. As the initial implementation, we can implement the kernel using Eigen following the abstraction pattern implemented for [Operators](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/rmsprop_op.h). We also want to support the case when the Trainer/Optimizer runs on the GPU while ParameterAverageOptimizer runs on a CPU. + +The idea of building an op for averaging is in sync with the refactored PaddlePaddle philosophy of using operators to represent any computation unit. The way the op will be added to the computation graph will be decided by the [layer functions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md#layer-function) in Python API. + +### Python API implementation for ParameterAverageOptimizer + +Based on Polyak and Juditsky (1992), we can generalize the averaging of updates to any optimizer. The input to the op would be the following: +- Any optimizer (RMSProp , AdaGrad etc.) +- A window size. The op keeps accumulating updated parameter values over a window of N batches and takes an average. Move the averaged value to a buffer when window is full to avoid loss of precision. + +Using the ParameterAverageOptimizer op, any user can add the operation to their computation graphs. However, this will require a lot of lines of code and we should design Python APIs that support averaging. As per the PaddlePaddle [Python API design](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md), the layer functions are responsible for creating operators, operator parameters and variables. Since ParameterAverageOptimizer will be an operator, it makes sense to create it in the layer functions. +We will have a wrapper written in Python that will support the functionality and implement the actual core computation in C++ core as we have done for other [Optimizers](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/rmsprop_op.cc) + +#### Creation of the ParameterAverageOptimizer operator +There are two ways for creating the ParameterAverageOptimizer op: +1. We create the op immediately while building the computation graph. +2. We add the op in a lazy manner, just before the backward pass, similar to the way the optimization ops are added. + +The proposal is to add the op immediately while building the computation graph. + +#### High-level API + +In PaddlePaddle Python API, users will primarily rely on [layer functions](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/python_api.md#layer-function) to create neural network layers. Hence, we also need to provide parameter average functionality in layer functions. diff --git a/doc/faq/local/index_cn.rst b/doc/faq/local/index_cn.rst index 0e939a2671ace8682c90cdc1c1bb2da1dda0d568..b331d9d36e6a279881c3b1a5586835e7186957fb 100644 --- a/doc/faq/local/index_cn.rst +++ b/doc/faq/local/index_cn.rst @@ -99,7 +99,7 @@ PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`spa 利用更多的计算资源 ++++++++++++++++++ -利用更多的计算资源可以分为一下几个方式来进行\: +利用更多的计算资源可以分为以下几个方式来进行\: * 单机CPU训练 diff --git a/doc/faq/parameter/index_cn.rst b/doc/faq/parameter/index_cn.rst index c721b623183cc7d8d17e2c9fb1635ea07b8970cc..6fa0c64413be1616a435640b0347904a49873349 100644 --- a/doc/faq/parameter/index_cn.rst +++ b/doc/faq/parameter/index_cn.rst @@ -75,7 +75,7 @@ PaddlePaddle目前支持8种learning_rate_schedule，这8种learning_rate_schedu optimizer = paddle.optimizer.Adam( learning_rate=1e-3, - learning_rate_schedule="manual", + learning_rate_schedule="pass_manual", learning_rate_args="1:1.0,2:0.9,3:0.8",) 在该示例中，当已训练pass数小于等于1时，学习率为 :code:`1e-3 * 1.0`；当已训练pass数大于1小于等于2时，学习率为 :code:`1e-3 * 0.9`；当已训练pass数大于2时，学习率为 :code:`1e-3 * 0.8`。 diff --git a/doc/getstarted/build_and_install/docker_install_cn.rst b/doc/getstarted/build_and_install/docker_install_cn.rst index 30b144d849bec367cd0197b6082889e011193a9a..0d34dec8e908c5e61001500725187a2233797f46 100644 --- a/doc/getstarted/build_and_install/docker_install_cn.rst +++ b/doc/getstarted/build_and_install/docker_install_cn.rst @@ -145,7 +145,7 @@ PaddlePaddle发布新版本的时候都会发布对应版本的生产镜像以 Jupyter Notebook是一个开源的web程序，大家可以通过它制作和分享带有代码、公式、图表、文字的交互式文档。用户可以通过网页浏览文档。 -PaddlePaddle Book是为用户和开发者制作的一个交互式的Jupyter Nodebook。 +PaddlePaddle Book是为用户和开发者制作的一个交互式的Jupyter Notebook。 如果您想要更深入了解deep learning，PaddlePaddle Book一定是您最好的选择。 我们提供可以直接运行PaddlePaddle Book的Docker镜像，直接运行： diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index c823d7e9fcd63dd7719ac1403952b03c2d2f03c0..6cfc9536f20e88571a9845a50be0341fe4d9f78b 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -214,7 +214,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, ```cpp // if use Eigen unsupported module before include head files - #define EIGEN_USE_GPU + // #define EIGEN_USE_GPU namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL(mul, ops::MulKernel); diff --git a/doc/howto/usage/cmd_parameter/arguments_cn.md b/doc/howto/usage/cmd_parameter/arguments_cn.md index f7aa525054468670f59309ddf9206af55bb77869..2dea231ca5487978d59a4d0a570431722ed6b3bf 100644 --- a/doc/howto/usage/cmd_parameter/arguments_cn.md +++ b/doc/howto/usage/cmd_parameter/arguments_cn.md @@ -63,7 +63,7 @@ -训练dot_period +训练dot_period √√ diff --git a/doc/index_cn.rst b/doc/index_cn.rst index 9279bac7f4b2898c18979630a8d6dfcb2dba70e0..ada51c2d73263898b2c748437f8eb0f30b537073 100644 --- a/doc/index_cn.rst +++ b/doc/index_cn.rst @@ -8,3 +8,4 @@ PaddlePaddle 文档 howto/index_cn.rst api/index_cn.rst faq/index_cn.rst + mobile/index_cn.rst diff --git a/doc/index_en.rst b/doc/index_en.rst index 64684b8b9b27e245c6b32ea28809d3bbce22fab9..23b64b6cadf776d44c4d0aa5a550ffe24be13b18 100644 --- a/doc/index_en.rst +++ b/doc/index_en.rst @@ -7,3 +7,4 @@ PaddlePaddle Documentation getstarted/index_en.rst howto/index_en.rst api/index_en.rst + mobile/index_en.rst diff --git a/doc/howto/cross_compiling/cross_compiling_for_android_cn.md b/doc/mobile/cross_compiling_for_android_cn.md similarity index 92% rename from doc/howto/cross_compiling/cross_compiling_for_android_cn.md rename to doc/mobile/cross_compiling_for_android_cn.md index 1fc58c37cc9151d5e4d99b939e30c29aa99e04f1..882066f23714f7ab3bba9199b5fa5ff2325ce849 100644 --- a/doc/howto/cross_compiling/cross_compiling_for_android_cn.md +++ b/doc/mobile/cross_compiling_for_android_cn.md @@ -1,7 +1,7 @@ # 构建Android平台上的PaddlePaddle库 用户可通过如下两种方式，交叉编译Android平台上适用的PaddlePaddle库： -- 基于Docker容器的编译方式 +- 基于Docker容器的编译方式 - 基于Linux交叉编译环境的编译方式 ## 基于Docker容器的编译方式 @@ -20,20 +20,42 @@ $ docker build -t username/paddle-android:dev . -f Dockerfile.android 构建好开发镜像后，即可使用开发镜像来编译Android版PaddlePaddle C-API库。 Android的Docker开发镜像向用户提供两个可配置的参数： -| Argument | Optional Values | Default | -|-----------------|-------------------------|---------| -|`ANDROID_ABI` |`armeabi-v7a, arm64-v8a` | `armeabi-v7a` | -|`ANDROID_API` |`>= 21` | `21` | + ++ + + + + + + + + + + + + + + + + + + + + + + +
ArgumentOptional ValuesDefault
ANDROID_ABIarmeabi-v7a, arm64-v8aarmeabi-v7a
ANDROID_API>= 2121
- 编译`armeabi-v7a`，`Android API 21`的PaddlePaddle库 -```bash -$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" username/paddle-android:dev -``` + ```bash + $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" username/paddle-android:dev + ``` -- 编译`arm64-v8a`，`Android API 21`的PaddlePaddle库 -```bash -$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=arm64-v8a" -e "ANDROID_API=21" username/paddle-android:dev -``` +- 编译`arm64-v8a`，`Android API 21`的PaddlePaddle库 + ```bash + $ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=arm64-v8a" -e "ANDROID_API=21" username/paddle-android:dev + ``` 执行上述`docker run`命令时，容器默认执行[paddle/scripts/docker/build_android.sh](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh)脚本。该脚本中记录了交叉编译Android版PaddlePaddle库常用的CMake配置，并且会根据`ANDROID_ABI`和`ANDROID_API`自动构建独立工具链、进行编译和安装。由于arm64架构要求Android API不小于21。因此当`ANDROID_ABI=arm64-v8a`，`ANDROID_API<21`时，Docker容器中将默认使用`Android API 21`的编译工具链。用户可以参考下文**配置交叉编译参数**章节，根据个人的需求修改定制Docker容器所执行的脚本。编译安装结束之后，PaddlePaddle的C-API库将被安装到`$PWD/install_android`目录，所依赖的第三方库同时也被安装到`$PWD/install_android/third_party`目录。 @@ -82,16 +104,16 @@ CMake系统对交叉编译提供了支持[cmake-toolchains](https://cmake.org/cm Android平台可选配置参数： - `ANDROID_STANDALONE_TOOLCHAIN`，独立工具链所在的绝对路径，或者相对于构建目录的相对路径。PaddlePaddle的CMake系统将根据该值自动推导和设置需要使用的交叉编译器、sysroot、以及Android API级别；否则，用户需要在cmake时手动设置这些值。无默认值。 -- `ANDROID_TOOLCHAIN`，目标工具链。可设置`gcc/clang`，默认值为`clang`。 - - CMake 3.7以上，将会始终使用`clang`工具链；CMake 3.7以下，可设置`ANDROID_TOOLCHAIN=gcc`以使用`gcc`工具链。 +- `ANDROID_TOOLCHAIN`，目标工具链。可设置`gcc/clang`，默认值为`clang`。 + - CMake 3.7以上，将会始终使用`clang`工具链；CMake 3.7以下，可设置`ANDROID_TOOLCHAIN=gcc`以使用`gcc`工具链。 - Android官方提供的`clang`编译器要求系统支持`GLIBC 2.15`以上。 - `ANDROID_ABI`，目标架构ABI。目前支持`armeabi-v7a`和`arm64-v8a`，默认值为`armeabi-v7a`。 - `ANDROID_NATIVE_API_LEVEL`，工具链的Android API级别。若没有显式设置，PaddlePaddle将根据`ANDROID_STANDALONE_TOOLCHAIN`的值自动推导得到。 -- `ANROID_ARM_MODE`，是否使用ARM模式。 - - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； +- `ANROID_ARM_MODE`，是否使用ARM模式。 + - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； - `ANDROID_ABI=arm64-v8a`时，不需要设置。 -- `ANDROID_ARM_NEON`，是否使用NEON指令。 - - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； +- `ANDROID_ARM_NEON`，是否使用NEON指令。 + - `ANDROID_ABI=armeabi-v7a`时，可设置`ON/OFF`，默认值为`ON`； - `ANDROID_ABI=arm64-v8a`时，不需要设置。 其他配置参数： @@ -119,7 +141,7 @@ cmake -DCMAKE_SYSTEM_NAME=Android \ -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm64_standalone_toolchain \ -DANDROID_ABI=arm64-v8a \ -DUSE_EIGEN_FOR_BLAS=OFF \ - -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ -DWITH_C_API=ON \ -DWITH_SWIG_PY=OFF \ .. @@ -128,8 +150,8 @@ cmake -DCMAKE_SYSTEM_NAME=Android \ 用户还可根据自己的需求设置其他编译参数。比如希望最小化生成的库的大小，可以设置`CMAKE_BUILD_TYPE`为`MinSizeRel`；若希望最快的执行速度，则可设置`CMAKE_BUILD_TYPE`为`Release`。亦可以通过手动设置`CMAKE_C/CXX_FLAGS_MINSIZEREL/RELEASE`来影响PaddlePaddle的编译过程。 **性能TIPS**，为了达到最快的计算速度，在CMake参数配置上，有以下建议： -- 设置`CMAKE_BUILD_TYPE`为`Release` -- 使用`clang`编译工具链 +- 设置`CMAKE_BUILD_TYPE`为`Release` +- 使用`clang`编译工具链 - `armeabi-v7a`时，设置`USE_EIGEN_BLAS=ON`，使用Eigen进行矩阵计算；`arm64-v8a`时，设置`USE_EIGEN_FOR_BLAS=OFF`，使用OpenBLAS进行矩阵计算 ### 编译和安装 diff --git a/doc/mobile/cross_compiling_for_android_en.md b/doc/mobile/cross_compiling_for_android_en.md new file mode 100644 index 0000000000000000000000000000000000000000..26858581fc1d77a9391520ac0dfd80fbd98f508c --- /dev/null +++ b/doc/mobile/cross_compiling_for_android_en.md @@ -0,0 +1,175 @@ +# Build PaddlePaddle for Android + +There are two approaches to build PaddlePaddle for Android: using Docker and on Linux without Docker. + +## Cross-Compiling Using Docker + +Docker-based cross-compiling is the recommended approach because Docker runs on all major operating systems, including Linux, Mac OS X, and Windows. + +### Build the Docker Image + +The following steps pack all the tools that we need to build PaddlePaddle into a Docker image. + +```bash +$ git clone https://github.com/PaddlePaddle/Paddle.git +$ cd Paddle +$ docker build -t paddle:dev-android . -f Dockerfile.android +``` + +### Build the Inference Library + +We can run the Docker image we just created to build the inference library of PaddlePaddle for Android using the command below: + +```bash +$ docker run -it --rm -v $PWD:/paddle -e "ANDROID_ABI=armeabi-v7a" -e "ANDROID_API=21" paddle:dev-android +``` + +The Docker image accepts two arguments `ANDROID_ABI` and `ANDROID_API`: + + ++ + + + + + + + + + + + + + + + + + + + + + + +
ArgumentOptional ValuesDefault
ANDROID_ABIarmeabi-v7a, arm64-v8aarmeabi-v7a
ANDROID_API>= 2121
+ +The ARM-64 architecture (`arm64-v8a`) requires at least level 21 of Android API. + +The default entry-point of the Docker image, [`paddle/scripts/docker/build_android.sh`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/scripts/docker/build_android.sh) generates the [Android cross-compiling standalone toolchain](https://developer.android.com/ndk/guides/standalone_toolchain.html) based on the argument: `ANDROID_ABI` or `ANDROID_API`. For information about other configuration arguments, please continue reading. + +The above command generates and outputs the inference library in `$PWD/install_android` and puts third-party libraries in `$PWD/install_android/third_party`. + +## Cross-Compiling on Linux + +The Linux-base approach to cross-compile is to run steps in `Dockerfile.android` manually on a Linux x64 computer. + +### Setup the Environment + +To build for Android's, we need [Android NDK]( +https://developer.android.com/ndk/downloads/index.html): + +```bash +wget -q https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip +unzip -q android-ndk-r14b-linux-x86_64.zip +``` + +Android NDK includes everything we need to build the [*standalone toolchain*](https://developer.android.com/ndk/guides/standalone_toolchain.html), which in then used to build PaddlePaddle for Android. (We plan to remove the intermediate stage of building the standalone toolchain in the near future.) + +- To build the standalone toolchain for `armeabi-v7a` and Android API level 21: + + ```bash + your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain.sh \ + --arch=arm --platform=android-21 --install-dir=your/path/to/arm_standalone_toolchain + ``` + + The generated standalone toolchain will be in `your/path/to/arm_standalone_toolchain`. + +- To build the standalone toolchain for `arm64-v8a` and Android API level 21: + + ```bash + your/path/to/android-ndk-r14b-linux-x86_64/build/tools/make-standalone-toolchain.sh \ + --arch=arm64 --platform=android-21 --install-dir=your/path/to/arm64_standalone_toolchain + ``` + + The generated standalone toolchain will be in `your/path/to/arm64_standalone_toolchain`. + +**Please be aware that the minimum level of Android API required by PaddlePaddle is 21.** + +### Cross-Compiling Arguments + +CMake supports [choosing the toolchain](https://cmake.org/cmake/help/v3.0/manual/cmake-toolchains.7.html#cross-compiling). PaddlePaddle provides [`android.cmake`](https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cross_compiling/android.cmake), which configures the Android cross-compiling toolchain for CMake. `android.cmake` is not required for CMake >= 3.7, which support Android cross-compiling. PaddlePaddle detects the CMake version, for those newer than 3.7, it uses [the official version](https://cmake.org/cmake/help/v3.7/manual/cmake-toolchains.7.html#cross-compiling). + +Some other CMake arguments you need to know: + +- `CMAKE_SYSTEM_NAME` must be `Android`. This tells PaddlePaddle's CMake system to cross-compile third-party dependencies. This also changes some other CMake arguments like `WITH_GPU=OFF`, `WITH_AVX=OFF`, `WITH_PYTHON=OFF`, and `WITH_RDMA=OFF`. +- `WITH_C_API` must be `ON`, to build the C-based inference library for Android. +- `WITH_SWIG_PY` must be `OFF` because the Android platform doesn't support SWIG-based API. + +Some Android-specific arguments: + +- `ANDROID_STANDALONE_TOOLCHAIN`: the absolute path of the Android standalone toolchain, or the path relative to the CMake build directory. PaddlePaddle's CMake extensions would derive the cross-compiler, sysroot and Android API level from this argument. +- `ANDROID_TOOLCHAIN`: could be `gcc` or `clang`. The default value is `clang`. + - For CMake >= 3.7, it should anyway be `clang`. For older versions, it could be `gcc`. + - Android's official `clang` requires `glibc` >= 2.15. +- `ANDROID_ABI`: could be `armeabi-v7a` or `arm64-v8a`. The default value is `armeabi-v7a`. +- `ANDROID_NATIVE_API_LEVEL`: could be derived from the value of `ANDROID_STANDALONE_TOOLCHAIN`. +- `ANROID_ARM_MODE`: + - could be `ON` or `OFF`, and defaults to `ON`, when `ANDROID_ABI=armeabi-v7a`; + - no need to specify when `ANDROID_ABI=arm64-v8a`. +- `ANDROID_ARM_NEON`: indicates if to use NEON instructions. + - could be `ON` or `OFF`, and defaults to `ON`, when `ANDROID_ABI=armeabi-v7a`; + - no need to specify when `ANDROID_ABI=arm64-v8a`. + +Other useful arguments: + +- `USE_EIGEN_FOR_BLAS`: indicates if using Eigen. Could be `ON` or `OFF`, defaults to `OFF`. +- `HOST_C/CXX_COMPILER`: specifies the host compiler, which is used to build the host-specific protoc and target-specific OpenBLAS. It defaults to the value of the environment variable `CC`, or `cc`. + +Some frequent configurations for your reference: + +```bash +cmake -DCMAKE_SYSTEM_NAME=Android \ + -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm_standalone_toolchain \ + -DANDROID_ABI=armeabi-v7a \ + -DANDROID_ARM_NEON=ON \ + -DANDROID_ARM_MODE=ON \ + -DUSE_EIGEN_FOR_BLAS=ON \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_SWIG_PY=OFF \ + .. +``` + +``` +cmake -DCMAKE_SYSTEM_NAME=Android \ + -DANDROID_STANDALONE_TOOLCHAIN=your/path/to/arm64_standalone_toolchain \ + -DANDROID_ABI=arm64-v8a \ + -DUSE_EIGEN_FOR_BLAS=OFF \ + -DCMAKE_INSTALL_PREFIX=your/path/to/install \ + -DWITH_C_API=ON \ + -DWITH_SWIG_PY=OFF \ + .. +``` + + +There are some other arguments you might want to configure. + +- `CMAKE_BUILD_TYPE=MinSizeRel` minimizes the size of library. +- `CMAKE_BUILD_TYPE-Release` optimizes the runtime performance. + +Our own tip for performance optimization to use clang and Eigen or OpenBLAS: +- `CMAKE_BUILD_TYPE=Release` +- `ANDROID_TOOLCHAIN=clang` +- `USE_EIGEN_BLAS=ON` for `armeabi-v7a`, or `USE_EIGEN_FOR_BLAS=OFF` for `arm64-v8a`. + +### Build and Install + +After running `cmake`, we can run `make; make install` to build and install. + +Before building, you might want to remove the `third_party` and `build` directories including pre-built libraries for other architectures. + +After building，in the directory `CMAKE_INSTALL_PREFIX`, you will find three sub-directories: + +- `include`: the header file of the inference library, +- `lib`: the inference library built for various Android ABIs, +- `third_party`: dependent third-party libraries built for Android. diff --git a/doc/howto/cross_compiling/cross_compiling_for_ios_cn.md b/doc/mobile/cross_compiling_for_ios_cn.md similarity index 91% rename from doc/howto/cross_compiling/cross_compiling_for_ios_cn.md rename to doc/mobile/cross_compiling_for_ios_cn.md index 32c490d9aa4202e17aa1784a45a317c5307b98ea..cda636a67de712e072f4cc7ad859dda75211eaa8 100644 --- a/doc/howto/cross_compiling/cross_compiling_for_ios_cn.md +++ b/doc/mobile/cross_compiling_for_ios_cn.md @@ -27,10 +27,28 @@ iOS平台可选配置参数： - `SIMULATOR`，构建目标为`x86`架构的模拟器平台。 - `IOS_ARCH`，目标架构。针对不同的`IOS_PLATFORM`，可设置的目标架构如下表所示： - | IOS_PLATFORM | IOS_ARCH | - |--------------|----------------------| - | OS | armv7, armv7s, arm64 (默认) | - | SIMULATOR | i386, x86_64 (默认) | + + + + + + + + + + + + + + + + + + + + + +
IOS_PLATFORMIOS_ARCH
OSarmv7, armv7s, arm64 (默认)
SIMULATORi386, x86_64 (默认)
- `IOS_DEPLOYMENT_TARGET`，最小的iOS部署版本，默认值为`7.0`。 - `IOS_ENABLE_BITCODE`，是否使能[Bitcode](https://developer.apple.com/library/content/documentation/IDEs/Conceptual/AppDistributionGuide/AppThinning/AppThinning.html#//apple_ref/doc/uid/TP40012582-CH35-SW3)，可设置`ON/OFF`，默认值为`ON`。 diff --git a/doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md b/doc/mobile/cross_compiling_for_raspberry_cn.md similarity index 100% rename from doc/howto/cross_compiling/cross_compiling_for_raspberry_cn.md rename to doc/mobile/cross_compiling_for_raspberry_cn.md diff --git a/doc/howto/cross_compiling/cross_compiling_for_raspberry_en.md b/doc/mobile/cross_compiling_for_raspberry_en.md similarity index 100% rename from doc/howto/cross_compiling/cross_compiling_for_raspberry_en.md rename to doc/mobile/cross_compiling_for_raspberry_en.md diff --git a/doc/mobile/index_cn.rst b/doc/mobile/index_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..1d99666e58b7043b85b0203ee0dfcd1957710161 --- /dev/null +++ b/doc/mobile/index_cn.rst @@ -0,0 +1,9 @@ +MOBILE +====== + +.. toctree:: + :maxdepth: 1 + + cross_compiling_for_android_cn.md + cross_compiling_for_ios_cn.md + cross_compiling_for_raspberry_cn.md diff --git a/doc/mobile/index_en.rst b/doc/mobile/index_en.rst new file mode 100644 index 0000000000000000000000000000000000000000..3c08d736717cfe8d5fdf449dc58015086befbe60 --- /dev/null +++ b/doc/mobile/index_en.rst @@ -0,0 +1,8 @@ +MOBILE +====== + +.. toctree:: + :maxdepth: 1 + + cross_compiling_for_android_en.md + cross_compiling_for_raspberry_en.md diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt index e767856d5012fd205f6b57f9721d0cbca8dc46ed..d267b14657be2a773d1dacfd9ac3767cddc47415 100644 --- a/paddle/capi/CMakeLists.txt +++ b/paddle/capi/CMakeLists.txt @@ -29,32 +29,32 @@ add_style_check_target(paddle_capi ${CAPI_SOURCES} ${CAPI_HEADER} add_dependencies(paddle_capi paddle_proto) # TODO: paddle_capi_whole will be removed. +set(PADDLE_CAPI_LAYERS_LIBS + paddle_function + paddle_gserver) if(MOBILE_INFERENCE) - set(PADDLE_CAPI_INFER_LIBS - paddle_utils - paddle_parameter - paddle_math - paddle_cuda - paddle_function - paddle_gserver - paddle_proto) + set(PADDLE_CAPI_ENGINE_LIBS + paddle_utils + paddle_parameter + paddle_math + paddle_cuda + paddle_proto) else() - set(PADDLE_CAPI_INFER_LIBS - paddle_utils - paddle_parameter - paddle_math - paddle_cuda - paddle_function - paddle_gserver - paddle_proto - paddle_pserver - paddle_network) + set(PADDLE_CAPI_ENGINE_LIBS + paddle_utils + paddle_parameter + paddle_math + paddle_cuda + paddle_proto + paddle_pserver + paddle_network) endif() +set(PADDLE_CAPI_INFER_LIBS ${PADDLE_CAPI_LAYERS_LIBS} ${PADDLE_CAPI_ENGINE_LIBS}) cc_library(paddle_capi_whole DEPS paddle_capi ${PADDLE_CAPI_INFER_LIBS}) # Link the static library for inference -cc_library(paddle_capi_engine DEPS paddle_capi paddle_utils paddle_parameter paddle_math paddle_cuda paddle_proto) -cc_library(paddle_capi_layers DEPS paddle_function paddle_gserver) +cc_library(paddle_capi_engine DEPS paddle_capi ${PADDLE_CAPI_ENGINE_LIBS}) +cc_library(paddle_capi_layers DEPS ${PADDLE_CAPI_LAYERS_LIBS}) # Link the shared library for inference if(NOT IOS) diff --git a/paddle/capi/Matrix.cpp b/paddle/capi/Matrix.cpp index 4547afaf1dc9af8bc7909a684db766fdd7b159c0..53a36f8f20d1143470928f57eda6f575d9048236 100644 --- a/paddle/capi/Matrix.cpp +++ b/paddle/capi/Matrix.cpp @@ -54,6 +54,46 @@ paddle_error paddle_matrix_set_row(paddle_matrix mat, return kPD_NO_ERROR; } +PD_API paddle_error paddle_matrix_set_value(paddle_matrix mat, + paddle_real* value) { + if (mat == nullptr || value == nullptr) return kPD_NULLPTR; + auto ptr = cast(mat); + if (ptr->mat == nullptr) return kPD_NULLPTR; + paddle::real* buf = ptr->mat->getRowBuf(0); + size_t width = ptr->mat->getWidth(); + size_t height = ptr->mat->getHeight(); + if (ptr->mat->useGpu()) { +#ifdef PADDLE_WITH_CUDA + hl_memcpy(buf, value, sizeof(paddle::real) * width * height); +#else + return kPD_NOT_SUPPORTED; +#endif + } else { + std::copy(value, value + width * height, buf); + } + return kPD_NO_ERROR; +} + +PD_API paddle_error paddle_matrix_get_value(paddle_matrix mat, + paddle_real* result) { + if (mat == nullptr || result == nullptr) return kPD_NULLPTR; + auto ptr = cast(mat); + if (ptr->mat == nullptr) return kPD_NULLPTR; + paddle::real* buf = ptr->mat->getRowBuf(0); + size_t width = ptr->mat->getWidth(); + size_t height = ptr->mat->getHeight(); + if (ptr->mat->useGpu()) { +#ifdef PADDLE_WITH_CUDA + hl_memcpy(result, buf, width * height * sizeof(paddle::real)); +#else + return kPD_NOT_SUPPORTED; +#endif + } else { + std::copy(buf, buf + width * height, result); + } + return kPD_NO_ERROR; +} + paddle_error paddle_matrix_get_row(paddle_matrix mat, uint64_t rowID, paddle_real** rawRowBuffer) { diff --git a/paddle/capi/examples/model_inference/dense/main.c b/paddle/capi/examples/model_inference/dense/main.c index 3e6bd5285058a297c4574631e2a5c033b83936e8..876af2aa7615c098d225b56ce2ea0b1529a6e3c6 100644 --- a/paddle/capi/examples/model_inference/dense/main.c +++ b/paddle/capi/examples/model_inference/dense/main.c @@ -27,18 +27,20 @@ int main() { CHECK(paddle_arguments_resize(in_args, 1)); // Create input matrix. - paddle_matrix mat = paddle_matrix_create(/* sample_num */ 1, + paddle_matrix mat = paddle_matrix_create(/* sample_num */ 10, /* size */ 784, /* useGPU */ false); srand(time(0)); - paddle_real* array; - // Get First row. - CHECK(paddle_matrix_get_row(mat, 0, &array)); + std::vector input; + input.resize(784 * 10); - for (int i = 0; i < 784; ++i) { - array[i] = rand() / ((float)RAND_MAX); + for (int i = 0; i < input.size(); ++i) { + input[i] = rand() / ((float)RAND_MAX); } + + // Set value for the input matrix + CHECK(paddle_matrix_set_value(mat, input.data())); CHECK(paddle_arguments_set_value(in_args, 0, mat)); @@ -51,11 +53,17 @@ int main() { CHECK(paddle_arguments_get_value(out_args, 0, prob)); - CHECK(paddle_matrix_get_row(prob, 0, &array)); + std::std::vector result; + int height; + int width; + + CHECK(paddle_matrix_get_shape(prob, &height, &width); + result.resize(height * width); + CHECK(paddle_matrix_get_value(prob, result.data())); printf("Prob: "); - for (int i = 0; i < 10; ++i) { - printf("%.2f ", array[i]); + for (int i = 0; i < height * width; ++i) { + printf("%.2f ", result[i]); } printf("\n"); diff --git a/paddle/capi/matrix.h b/paddle/capi/matrix.h index f15f7f3bbbd1457617111f827d2182ae6b7d9fdb..bb5223f8a275fa2550bf8b7e94a9c4333de4c8c9 100644 --- a/paddle/capi/matrix.h +++ b/paddle/capi/matrix.h @@ -70,6 +70,16 @@ PD_API paddle_error paddle_matrix_set_row(paddle_matrix mat, uint64_t rowID, paddle_real* rowArray); +/** + * @brief paddle_matrix_set_value Set value to matrix. + * @param mat Target Matrix + * @param value Row data. + * @return paddle_error + * @note value should contain enough element of data to init the mat + */ +PD_API paddle_error paddle_matrix_set_value(paddle_matrix mat, + paddle_real* value); + /** * @brief PDMatGetRow Get raw row buffer from matrix * @param [in] mat Target matrix @@ -81,6 +91,15 @@ PD_API paddle_error paddle_matrix_get_row(paddle_matrix mat, uint64_t rowID, paddle_real** rawRowBuffer); +/** + * @brief copy data from the matrix + * @param [in] mat Target matrix + * @param [out] result pointer to store the matrix data + * @return paddle_error + * @note the space of the result should allocated before invoke this API + */ +PD_API paddle_error paddle_matrix_get_value(paddle_matrix mat, + paddle_real* result); /** * @brief PDMatCreateNone Create None Matrix * @return diff --git a/paddle/capi/tests/test_Matrix.cpp b/paddle/capi/tests/test_Matrix.cpp index 4bf9a9d6a9f9161561e9e5612edd2c93cab7ac5b..6940c28448a897cecd78b718fe720441086a5a99 100644 --- a/paddle/capi/tests/test_Matrix.cpp +++ b/paddle/capi/tests/test_Matrix.cpp @@ -45,3 +45,49 @@ TEST(CAPIMatrix, createNone) { paddle_matrix mat = paddle_matrix_create_none(); ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); } + +TEST(CAPIMatrix, cpu_get_set_value) { + paddle_matrix mat = paddle_matrix_create(128, 32, false); + std::vector sample; + std::vector result; + sample.resize(128 * 32); + result.resize(128 * 32); + for (size_t i = 0; i < sample.size(); ++i) { + sample[i] = 1.0 / (i + 1.0); + } + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_set_value(mat, sample.data())); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_value(mat, result.data())); + for (size_t i = 0; i < sample.size(); ++i) { + ASSERT_NEAR(sample[i], result[i], 1e-5); + } + + uint64_t height, width; + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_shape(mat, &height, &width)); + ASSERT_EQ(128UL, height); + ASSERT_EQ(32UL, width); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); +} + +#ifdef PADDLE_WITH_CUDA +TEST(CAPIMatrix, gpu_get_set_value) { + paddle_matrix mat = paddle_matrix_create(128, 32, true); + std::vector sample; + std::vector result; + sample.resize(128 * 32); + result.resize(128 * 32); + for (size_t i = 0; i < sample.size(); ++i) { + sample[i] = 1.0 / (i + 1.0); + } + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_set_value(mat, sample.data())); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_value(mat, result.data())); + for (size_t i = 0; i < sample.size(); ++i) { + ASSERT_NEAR(sample[i], result[i], 1e-5); + } + + uint64_t height, width; + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_get_shape(mat, &height, &width)); + ASSERT_EQ(128UL, height); + ASSERT_EQ(32UL, width); + ASSERT_EQ(kPD_NO_ERROR, paddle_matrix_destroy(mat)); +} +#endif diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index c7f25109972195fb56b9e96c4b68d952363e6338..7daca18761b80eac0f876b21377a6ccc6a853485 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -300,4 +300,12 @@ extern void hl_matrix_col2Vol(real* dataDst, real alpha, real beta); +/** + * @brief Matrix col2Vol: Convert col matrix into 3D volume + * @param[out] out output int vector. + * @param[in] vec input float vector. + * @param[in] size size of the vector. + */ +extern void hl_vector_cast2int(int* out, real* vec, int size); + #endif /* HL_MATRIX_H_ */ diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index 6ac332945c8f09fef23f35680ba5bb1d9ba9f4fd..46e77e140768dd80fd327dd4eb3b0f62a3370950 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -133,4 +133,6 @@ inline void hl_matrix_col2Vol(real* dataDst, real alpha, real beta) {} +inline void hl_vector_cast2int(int* out, real* vec, int size) {} + #endif // HL_MATRIX_STUB_H_ diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index b41a3a1e06db7b2566acef19ce430645f79d486d..607efb4f6b0aa0d22a2789397b8743f7a5271d5b 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -793,3 +793,14 @@ void hl_matrix_col2Vol(real* dataDst, CHECK_SYNC("hl_matrix_col2Vol failed"); } + +__global__ void keVectorCast2Int(int* out, real* vec, int size) { + for (int i = threadIdx.x; i < (size); i += blockDim.x) { + out[i] = int(vec[i]); + } +} + +void hl_vector_cast2int(int* out, real* vec, int size) { + keVectorCast2Int<<<1, 512, 0, STREAM_DEFAULT>>>(out, vec, size); + CHECK_SYNC("hl_vector_cast2int failed"); +} diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index f4fef055daf39e9be0645deaafdad4132fc7e35f..1afc5242081e7f7b12527a15d29421cebeb3d3b8 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -20,7 +20,8 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) cc_library(attribute SRCS attribute.cc DEPS framework_proto) -cc_test(program_desc_test SRCS program_desc_test.cc DEPS proto_desc) +cc_test(program_desc_test SRCS program_desc_test.cc DEPS proto_desc +device_context) cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) @@ -44,8 +45,9 @@ add_custom_command(TARGET framework_py_proto POST_BUILD cc_library(backward SRCS backward.cc DEPS net_op) cc_test(backward_test SRCS backward_test.cc DEPS backward recurrent_op device_context fill_constant_op) +cc_library(lod_rank_table SRCS lod_rank_table.cc DEPS lod_tensor) -cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog) +cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward glog lod_rank_table) cc_library(prune SRCS prune.cc DEPS framework_proto) cc_test(prune_test SRCS prune_test.cc DEPS op_info prune recurrent_op device_context) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index dbd5a14f9f3b681f0b77b9bd507b34edfaa78766..b3b9c45ded95ce2e735b8898d47760956dcacdce 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -24,7 +24,6 @@ #include "paddle/framework/op_registry.h" #include "paddle/operators/dynamic_recurrent_op.h" #include "paddle/operators/net_op.h" -#include "paddle/operators/recurrent_op.h" namespace paddle { namespace framework { @@ -38,7 +37,7 @@ static inline std::unique_ptr CreateGradOp( op_desc.SetType(op.Type()); op_desc.SetAttrMap(op.Attrs()); auto& info = OpInfoMap::Instance().Get(op.Type()); - auto grad_descs = info.GradOpMaker()(op_desc, no_grad_set, grad_to_var); + auto grad_descs = info.GradOpMaker()(op_desc, no_grad_set, grad_to_var, {}); std::vector> grad_ops; grad_ops.reserve(grad_descs.size()); std::transform(grad_descs.begin(), grad_descs.end(), @@ -220,19 +219,7 @@ static std::unique_ptr BackwardRecursive( }); // process recurrent gradient op as a special operator. - if (forwardOp.Type() == "recurrent") { - // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), - // or this will result in infinite loop. - const auto& rnnop = - *static_cast(&forwardOp); - auto rnn_grad_op = - static_cast(grad_op.get()); - const auto& stepnet_op = - *static_cast(&rnnop.stepnet()); - // create stepnet's gradient op - rnn_grad_op->set_stepnet( - BackwardRecursive(stepnet_op, no_grad_names, grad_to_var, uniq_id)); - } else if (forwardOp.Type() == "dynamic_recurrent") { + if (forwardOp.Type() == "dynamic_recurrent") { // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), // or this will result in infinite loop. const auto& rnnop = @@ -331,11 +318,9 @@ static void CreateGradVarInBlock( continue; } auto pname = FwdName(arg); - auto* param = block_desc->FindVar(pname); + auto* param = block_desc->FindVarRecursive(pname); auto* grad = block_desc->FindVar(arg); if (param == nullptr) { - LOG(WARNING) << "Cannot find forward variable of " << arg - << ". Set its gradient to FP32"; grad->SetDataType(DataType::FP32); } else { grad->SetDataType(param->GetDataType()); @@ -348,7 +333,9 @@ static void CreateGradVarInBlock( std::vector> MakeOpGrad( const OpDescBind* op_desc, std::unordered_set* no_grad_vars, - std::unordered_map* grad_to_var) { + std::unordered_map* grad_to_var, + const std::vector& grad_block = + std::vector()) { std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculate. const std::vector& inputs = op_desc->InputArgumentNames(); @@ -364,9 +351,10 @@ std::vector> MakeOpGrad( return grad_op_descs; // empty vector } - grad_op_descs = OpInfoMap::Instance() - .Get(op_desc->Type()) - .GradOpMaker()(*op_desc, *no_grad_vars, grad_to_var); + grad_op_descs = + OpInfoMap::Instance() + .Get(op_desc->Type()) + .GradOpMaker()(*op_desc, *no_grad_vars, grad_to_var, grad_block); std::list> pending_fill_zeros_ops; for (auto& desc : grad_op_descs) { @@ -389,6 +377,12 @@ std::vector> MakeOpGrad( return grad_op_descs; } +static BlockDescBind* CreateStepBlock( + ProgramDescBind& program_desc, + std::unordered_set* no_grad_vars, + std::unordered_map* grad_to_var, + int step_block_idx); + std::vector> MakeBlockBackward( ProgramDescBind& program_desc, int block_idx, std::unordered_set* no_grad_vars, @@ -400,25 +394,29 @@ std::vector> MakeBlockBackward( std::vector> backward_descs; for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { - std::vector> op_grads = - MakeOpGrad(*it, no_grad_vars, grad_to_var); + std::vector> op_grads; if ((*it)->Type() == "recurrent") { - PADDLE_ENFORCE_EQ( - op_grads.size(), static_cast(1), - "rnn_op's gradient process should contain only one op."); int step_block_idx = (*it)->GetBlockAttr("step_block"); - auto backward_block_op_descs = MakeBlockBackward( - program_desc, step_block_idx, no_grad_vars, grad_to_var); - BlockDescBind* backward_block = program_desc.AppendBlock(*cur_block); - for (auto& ptr : backward_block_op_descs) { - backward_block->AppendAllocatedOp(std::move(ptr)); - } - op_grads[0]->SetBlockAttr("step_block", *backward_block); + BlockDescBind* backward_block = CreateStepBlock( + program_desc, no_grad_vars, grad_to_var, step_block_idx); + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); + } else if ((*it)->Type() == "conditional_block") { + BlockDescBind* backward_block = + CreateStepBlock(program_desc, no_grad_vars, grad_to_var, + (*it)->GetBlockAttr("block")); + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var, {backward_block}); + } else { + op_grads = MakeOpGrad(*it, no_grad_vars, grad_to_var); } for (const auto& desc : op_grads) { for (const std::string& out_name : desc->OutputArgumentNames()) { + if (out_name.find("@GRAD") == std::string::npos) { + // Not all outputs of a backward operator is a gradient. Only gradient + // need to be sum. Skip variables are not gradient. + continue; + } dup_out_ops[out_name].emplace_back(grad_desc_idx); } ++grad_desc_idx; @@ -457,6 +455,21 @@ std::vector> MakeBlockBackward( return backward_descs; } +static BlockDescBind* CreateStepBlock( + ProgramDescBind& program_desc, + std::unordered_set* no_grad_vars, + std::unordered_map* grad_to_var, + int step_block_idx) { + auto backward_block_op_descs = MakeBlockBackward(program_desc, step_block_idx, + no_grad_vars, grad_to_var); + BlockDescBind* backward_block = + program_desc.AppendBlock(*program_desc.MutableBlock(step_block_idx)); + for (auto& ptr : backward_block_op_descs) { + backward_block->AppendAllocatedOp(move(ptr)); + } + return backward_block; +} + ParamGradInfoMap AppendBackward( ProgramDescBind& program_desc, const VarDescBind& target, const std::unordered_set& no_grad_vars) { diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 4e8d630c2634682ff63b38182108eadebb5c7ff9..d485cdf6109274377ad0057223bdd8401e964aa7 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -21,7 +21,7 @@ #include "paddle/framework/var_desc.h" #include "paddle/operators/net_op.h" -USE_OP(fill_constant); +USE_NO_KERNEL_OP(fill_constant); namespace paddle { namespace framework { diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 9e3d597f3a2c84623a1ce9e4b6f4b956cffde211..11764810e1d40e5e6eb3cd0d8e9b4b63a79855b4 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -50,6 +50,15 @@ VarDescBind *BlockDescBind::FindVarRecursive(const std::string &name) const { return it->second.get(); } +VarDescBind *BlockDescBind::FindRecursiveOrCreateVar( + const std::string &name_bytes) { + VarDescBind *res = FindVarRecursive(name_bytes); + if (res == nullptr) { + res = Var(name_bytes); + } + return res; +} + bool BlockDescBind::HasVarRecursive(const std::string &name) const { return FindVarRecursive(name) != nullptr; } diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 72f77a88a24434fd7d2ed685ac850c88888d6808..8e967e5378eb47a7869efb59cc96a271f1cbb9a1 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -58,6 +58,8 @@ class BlockDescBind { VarDescBind *FindVarRecursive(const std::string &name_bytes) const; + VarDescBind *FindRecursiveOrCreateVar(const std::string &name_bytes); + bool HasVarRecursive(const std::string &var_name) const; std::set LocalVarNames() const { @@ -88,6 +90,8 @@ class BlockDescBind { BlockDesc *Proto(); + ProgramDescBind *Program() { return this->prog_; } + private: void ClearPBOps(); void ClearPBVars(); diff --git a/paddle/framework/data_type.h b/paddle/framework/data_type.h index c5ae7b185460c8b0d68ba38bb9db9bd3d3fb14ea..3ec88d7a72c3339bf5e7d0ca3957a3f608f039b7 100644 --- a/paddle/framework/data_type.h +++ b/paddle/framework/data_type.h @@ -34,6 +34,21 @@ inline DataType ToDataType(std::type_index type) { } } +inline std::type_index ToTypeIndex(DataType type) { + switch (type) { + case DataType::FP32: + return typeid(float); + case DataType::FP64: + return typeid(double); + case DataType::INT32: + return typeid(int); + case DataType::INT64: + return typeid(int64_t); + default: + PADDLE_THROW("Not support type %d", type); + } +} + template inline void VisitDataType(DataType type, Visitor visitor) { switch (type) { diff --git a/paddle/framework/ddim.cc b/paddle/framework/ddim.cc index 239ae5e1233c7f5c506930df374b5d0cc8de7c8d..53b899a23997b71e723a298ec360a4e018d89878 100644 --- a/paddle/framework/ddim.cc +++ b/paddle/framework/ddim.cc @@ -79,6 +79,13 @@ DDim make_ddim(const std::vector& dims) { return result; } +DDim make_ddim(const std::vector& dims) { + std::vector res(dims.size()); + std::transform(dims.begin(), dims.end(), res.begin(), + [](int d) { return static_cast(d); }); + return make_ddim(res); +} + /// @cond HIDDEN // XXX For some reason, putting this in an anonymous namespace causes errors class DynamicMutableIndexer : public boost::static_visitor { @@ -117,7 +124,7 @@ int64_t DDim::operator[](int idx) const { return boost::apply_visitor(DynamicConstIndexer(idx), var); } -int64_t DDim::size() const { return arity(*this); } +int DDim::size() const { return arity(*this); } bool DDim::operator==(DDim d) const { if (var.which() != d.getVar().which()) { diff --git a/paddle/framework/ddim.h b/paddle/framework/ddim.h index 2a5e2d2b6948b045642dbac5e83992a048ecb63d..4ca5e49566b7ec006eba80f3f9808bacb1ff2615 100644 --- a/paddle/framework/ddim.h +++ b/paddle/framework/ddim.h @@ -71,7 +71,7 @@ struct DDim { DDim operator*(DDim d) const; - int64_t size() const; + int size() const; }; /** @@ -81,6 +81,8 @@ struct DDim { */ DDim make_ddim(const std::vector& dims); +DDim make_ddim(const std::vector& dims); + /** * \brief Make a DDim from an initializer list * diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index b731840ef2a4b2d5d82b019d28ad6517fa4b7607..f91e0e03410c95f84a65f02beed38b7bbfdcaa86 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -108,8 +108,9 @@ struct OpInfoFiller { info->grad_op_maker_ = []( const OpDescBind& fwd_op, const std::unordered_set& no_grad_set, - std::unordered_map* grad_to_var) { - T maker(fwd_op, no_grad_set, grad_to_var); + std::unordered_map* grad_to_var, + const std::vector& grad_block) { + T maker(fwd_op, no_grad_set, grad_to_var, grad_block); return maker(); }; } diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc index 9bf2311dc835c701c9311880b8adba486a7d446c..2fcf41d69f0011b0d9a3d89c97fcebacb0703e97 100644 --- a/paddle/framework/executor.cc +++ b/paddle/framework/executor.cc @@ -21,7 +21,9 @@ limitations under the License. */ #include #include "paddle/framework/feed_fetch_type.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/op_registry.h" #include "paddle/framework/scope.h" @@ -31,7 +33,7 @@ namespace framework { const std::string kFeedOpType = "feed"; const std::string kFetchOpType = "fetch"; -Executor::Executor(const std::vector& places) { +Executor::Executor(const std::vector& places) : own_(true) { PADDLE_ENFORCE_GT(places.size(), 0); device_contexts_.resize(places.size()); for (size_t i = 0; i < places.size(); i++) { @@ -52,8 +54,10 @@ Executor::Executor(const std::vector& places) { } Executor::~Executor() { - for (auto& device_context : device_contexts_) { - delete device_context; + if (own_) { + for (auto& device_context : device_contexts_) { + delete device_context; + } } } @@ -66,44 +70,65 @@ static void CreateTensor(Variable* var, VarDesc::VarType var_type) { var->GetMutable(); } else if (var_type == VarDesc::FETCH_LIST) { var->GetMutable(); + } else if (var_type == VarDesc::STEP_SCOPES) { + var->GetMutable>(); + } else if (var_type == VarDesc::LOD_RANK_TABLE) { + var->GetMutable(); + } else if (var_type == VarDesc::LOD_TENSOR_ARRAY) { + var->GetMutable(); } else { PADDLE_THROW( - "Variable type must be " - "LoDTensor/SelectedRows/FEED_MINIBATCH/FETCH_LIST."); + "Variable type %d is not in " + "[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST, LOD_RANK_TABLE]", + var_type); } } -void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id) { +void Executor::Run(const ProgramDescBind& pdesc, Scope* scope, int block_id, + bool create_local_scope) { // TODO(tonyyang-svail): // - only runs on the first device (i.e. no interdevice communication) // - will change to use multiple blocks for RNN op and Cond Op - PADDLE_ENFORCE_LT(block_id, pdesc.Size()); + PADDLE_ENFORCE_LT(static_cast(block_id), pdesc.Size()); auto& block = pdesc.Block(block_id); auto& device = device_contexts_[0]; - Scope& local_scope = scope->NewScope(); - - for (auto& var : block.AllVars()) { - if (var->Persistable()) { - auto* ptr = scope->Var(var->Name()); - CreateTensor(ptr, var->GetType()); - VLOG(3) << "Create Variable " << var->Name() - << " global, which pointer is " << ptr; - } else { - auto* ptr = local_scope.Var(var->Name()); + Scope* local_scope = scope; + if (create_local_scope) { + local_scope = &scope->NewScope(); + for (auto& var : block.AllVars()) { + if (var->Persistable()) { + auto* ptr = scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " global, which pointer is " << ptr; + } else { + auto* ptr = local_scope->Var(var->Name()); + CreateTensor(ptr, var->GetType()); + VLOG(3) << "Create Variable " << var->Name() + << " locally, which pointer is " << ptr; + } + } + } else { + for (auto& var : block.AllVars()) { + auto* ptr = local_scope->Var(var->Name()); CreateTensor(ptr, var->GetType()); - VLOG(3) << "Create Variable " << var->Name() - << " locally, which pointer is " << ptr; + VLOG(3) << "Create variable " << var->Name() << ", which pointer is " + << ptr; } } for (auto& op_desc : block.AllOps()) { auto op = paddle::framework::OpRegistry::CreateOp(*op_desc); - op->Run(local_scope, *device); + op->Run(*local_scope, *device); + } + if (create_local_scope) { + scope->DeleteScope(local_scope); } - - scope->DeleteScope(&local_scope); } +Executor::Executor(const platform::DeviceContext& device) + : device_contexts_({&device}), own_(false) {} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h index c78bfe8f9f07f1324515f0baaca4a94cc0fe844e..b745f4f6474ef688774f4c833a3958942e9aa8cb 100644 --- a/paddle/framework/executor.h +++ b/paddle/framework/executor.h @@ -25,6 +25,7 @@ namespace framework { class Executor { public: explicit Executor(const std::vector& places); + explicit Executor(const platform::DeviceContext& devices); ~Executor(); /* @Brief @@ -34,10 +35,11 @@ class Executor { * ProgramDesc * Scope */ - void Run(const ProgramDescBind&, Scope*, int); + void Run(const ProgramDescBind&, Scope*, int, bool create_local_scope = true); private: - std::vector device_contexts_; + std::vector device_contexts_; + bool own_; }; } // namespace framework diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 8f2df3dc0e29f96b3aea58b6761d1ccb4cd7c624..f1fc4529e15502927560eefd74110f6ca7eab4a9 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -109,6 +109,11 @@ message LoDTensorDesc { optional int32 lod_level = 2 [ default = 0 ]; } +message LoDTensorArrayDesc { + required TensorDesc tensor = 1; + optional int32 lod_level = 2 [ default = 0 ]; +} + message VarDesc { enum VarType { LOD_TENSOR = 1; @@ -116,11 +121,14 @@ message VarDesc { FEED_MINIBATCH = 3; FETCH_LIST = 4; STEP_SCOPES = 5; + LOD_RANK_TABLE = 6; + LOD_TENSOR_ARRAY = 7; } required string name = 1; required VarType type = 2; optional LoDTensorDesc lod_tensor = 3; optional TensorDesc selected_rows = 4; + optional LoDTensorArrayDesc tensor_array = 6; optional bool persistable = 5 [ default = false ]; } diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index 94944c79b64d38e799df436de874cabc3661e30a..998186e33915a11f2864eb5387d19ed1bfbab51c 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -15,6 +15,7 @@ #pragma once #include #include +#include #include "paddle/framework/op_desc.h" #include "paddle/framework/operator.h" @@ -26,8 +27,13 @@ class GradOpDescMakerBase { explicit GradOpDescMakerBase( const OpDescBind& fwd_op, const std::unordered_set& no_grad_set, - std::unordered_map* grad_to_var) - : fwd_op_(fwd_op), no_grad_set_(no_grad_set), grad_to_var_(grad_to_var) {} + std::unordered_map* grad_to_var, + const std::vector& grad_block = + std::vector()) + : fwd_op_(fwd_op), + no_grad_set_(no_grad_set), + grad_to_var_(grad_to_var), + grad_block_(grad_block) {} virtual ~GradOpDescMakerBase() = default; virtual std::vector> operator()() const = 0; @@ -102,6 +108,9 @@ class GradOpDescMakerBase { const OpDescBind& fwd_op_; const std::unordered_set& no_grad_set_; std::unordered_map* grad_to_var_; + + protected: + std::vector grad_block_; }; class SingleGradOpDescMaker : public GradOpDescMakerBase { diff --git a/paddle/framework/lod_rank_table.cc b/paddle/framework/lod_rank_table.cc new file mode 100644 index 0000000000000000000000000000000000000000..1c2fba70c8ab0827ba6d1563f08cd0820650822e --- /dev/null +++ b/paddle/framework/lod_rank_table.cc @@ -0,0 +1,49 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/lod_rank_table.h" + +namespace paddle { +namespace framework { +void LoDRankTable::Reset(const LoD& lod, size_t level) { + this->coarse_lod_.clear(); + this->items_.clear(); + PADDLE_ENFORCE(level < lod.size(), + "Cannot rank lod since the level %d is less than lod size %d", + level, lod.size()); + coarse_lod_.reserve(level); + for (size_t i = 0; i < level; ++i) { + coarse_lod_.push_back(lod[i]); + } + auto& vec = lod[level]; + for (size_t i = 0; i < vec.size() - 1; ++i) { + TableItem item; + item.index = i; + item.length = vec[i + 1] - vec[i]; + VLOG(10) << "Add item to rank table " << item.index << " " << item.length; + items_.emplace_back(item); + } + // NOTE(yuyang18): + // + // The time complexity of stable_sort is O(N*log(N)) if additional memory is + // available. It is easy to debug and unit test when using `stable_sort` + // instead of `sort`. Also, the items of a rank table will not be too large. + std::stable_sort(items_.begin(), items_.end(), + [](const TableItem& a, const TableItem& b) { + return a.length > b.length; + }); +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_rank_table.h b/paddle/framework/lod_rank_table.h new file mode 100644 index 0000000000000000000000000000000000000000..9faa3a4d7bdc55ab7b24e31f5e5434dacc0a4b36 --- /dev/null +++ b/paddle/framework/lod_rank_table.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { + +// LoD Rank Table stores the `level` of `lod` which is ordered by sequence +// length in descending order. It is useful when implement dynamic RNN and is +// shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +// output operators. +// +// The table item contains two element. The length of sequence and the index of +// sequence in that level. +// +// LoDRankTable also stores the coarse_lod, which is the lod information whose +// level is less than input level, in order to restore the output LoD +// information. +class LoDRankTable { + public: + struct TableItem { + size_t index; + size_t length; + }; + + LoDRankTable() {} + + void Reset(const LoD& lod, size_t level); + + const std::vector& items() const { return this->items_; } + + const LoD& coarse_lod() const { return this->coarse_lod_; } + + size_t level() const { return coarse_lod_.size(); } + + private: + LoD coarse_lod_; + std::vector items_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 584308a5388da0d02d29f71a28097b02b6ea825f..a0f2906c749054c1ff9f624e47df432ec2bd6ac8 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -27,6 +27,20 @@ namespace paddle { namespace framework { +std::ostream& operator<<(std::ostream& os, const LoD& lod) { + os << "{"; + for (auto& v : lod) { + os << "{"; + for (auto& i : v) { + os << i << ","; + } + os << "}"; + } + os << "}"; + + return os; +} + LoD SliceLevels(const LoD& in, size_t level_begin, size_t level_end) { LoD new_lod; new_lod.reserve(level_end - level_begin); @@ -135,5 +149,41 @@ void LoDTensor::ShrinkInLevel(size_t level, size_t elem_begin, PADDLE_ENFORCE_LT(begin, end, "Cannot shrink, the result tensor is empty."); ShareDataWith(Slice(begin, end)); } + +using LoDAndOffset = std::pair>; +LoDAndOffset GetSubLoDAndAbsoluteOffset(const LoD& lod, size_t start_idx, + size_t end_idx, size_t start_level) { + LoD sub_lod; + + for (size_t level_idx = start_level; level_idx < lod.size(); ++level_idx) { + PADDLE_ENFORCE_LE(start_idx, end_idx); + PADDLE_ENFORCE_LT(end_idx, lod[level_idx].size()); + std::vector level_lens; + for (size_t i = start_idx; i < end_idx; ++i) { + level_lens.push_back(lod[level_idx][i + 1] - lod[level_idx][i]); + } + sub_lod.emplace_back(level_lens); + start_idx = lod[level_idx][start_idx]; + end_idx = lod[level_idx][end_idx]; + } + + return LoDAndOffset{sub_lod, {start_idx, end_idx}}; +} + +void AppendLoD(LoD* lod, const LoD& lod_length) { + PADDLE_ENFORCE( + lod->empty() || lod->size() == lod_length.size(), + "The lod_length should has the same size with the appended lod."); + if (lod->empty()) { + *lod = LoD(lod_length.size(), std::vector({0})); + } + for (size_t i = 0; i < lod->size(); ++i) { + auto& level = (*lod)[i]; + for (size_t len : lod_length[i]) { + level.push_back(level.back() + len); + } + } +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index f4fe4cdac6019a1899fd3db8e1b6ca588be0d436..7f8a51cc581e759bc707e506ac7cdeb3680f40ac 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -56,6 +56,8 @@ using Vector = thrust::host_vector< */ using LoD = std::vector>; +std::ostream& operator<<(std::ostream& os, const LoD& lod); + /* * Slice levels from a LoD. * NOTE the lowest level should always be the absolute offsets of the underlying @@ -181,5 +183,10 @@ LoDTensor LodExpand(const LoDTensor& source, const LoD& lod, size_t level, return tensor; } +std::pair> GetSubLoDAndAbsoluteOffset( + const LoD& lod, size_t start_idx, size_t end_idx, size_t start_level); + +void AppendLoD(LoD* lod, const LoD& lod_length); + } // namespace framework } // namespace paddle diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index d147f1c4257eec14664301edab8d1fe2f128d2b0..10a8a7867fbf072f585fe3bfb1243e4e6bef4ec8 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -140,19 +140,9 @@ Similarly, the lengths in the top level LoD are transformed into offsets of elements/words as follows: ``` -0 9 10 15 - = = = - 3+2+4 1+9 2+3+10 -``` - -so we can tell that the first article is from word 0 to word 9, and the second article is from word 9 to word 10. - -The complete offset representation is as follows: - -``` -0 9 10 15 -0 3 5 9 10 12 15 - ||| || |||| | || ||| +0 3 4 6 + = = = + 3 3+1 4+2 ``` ## Slicing of LoD Tensors diff --git a/paddle/framework/lod_tensor_array.h b/paddle/framework/lod_tensor_array.h new file mode 100644 index 0000000000000000000000000000000000000000..13f0608d24be97d8bba149b74f1a4deb57deeb48 --- /dev/null +++ b/paddle/framework/lod_tensor_array.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { +using LoDTensorArray = std::vector; +} +} // namespace paddle diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index aa2f6c993d41ae98e0769d470dccad3b410da53e..02d84b68233f2fdfc66e1df2fc7ce20307cadd94 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -144,5 +144,48 @@ TEST(LodExpand, test) { } } +TEST(LoD, GetFineGrainedLoDLength) { + LoD lod; + lod.push_back(std::vector({0, 2, 4, 5})); + lod.push_back(std::vector({0, 1, 6, 8, 10, 11})); + lod.push_back( + std::vector({0, 2, 5, 7, 10, 12, 15, 17, 20, 24, 26, 29})); + + auto lod_and_offset = + paddle::framework::GetSubLoDAndAbsoluteOffset(lod, 1, 2, 0); + LoD lod_length = lod_and_offset.first; + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + + LoD expected; + expected.push_back(std::vector{2}); + expected.push_back(std::vector{2, 2}); + expected.push_back(std::vector{2, 3, 4, 2}); + EXPECT_EQ(lod_length, expected); + EXPECT_EQ(start_offset, 15UL); + EXPECT_EQ(end_offset, 26UL); +} + +TEST(LoD, AppendLoD) { + LoD lod_lens; + lod_lens.push_back(std::vector({2})); + lod_lens.push_back(std::vector({2, 2})); + lod_lens.push_back(std::vector({2, 3, 4, 2})); + + LoD origin; + origin.push_back(std::vector({0, 2})); + origin.push_back(std::vector({0, 1, 6})); + origin.push_back(std::vector({0, 2, 5, 7, 10, 12, 15})); + + paddle::framework::AppendLoD(&origin, lod_lens); + + LoD expected; + expected.push_back(std::vector({0, 2, 4})); + expected.push_back(std::vector({0, 1, 6, 8, 10})); + expected.push_back( + std::vector({0, 2, 5, 7, 10, 12, 15, 17, 20, 24, 26})); + EXPECT_EQ(origin, expected); +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 0779137639e6cd9f6ecf3bbbc24d081cae3de9c0..39c8def82e1ebb10a0e357a648af760099020c32 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -67,8 +67,11 @@ class CompileTimeInferShapeContext : public InferShapeContext { out); in_var->SetLoDLevel(out_var->GetLodLevel()); } + bool IsRuntime() const override; + + protected: + VarDesc::VarType GetVarType(const std::string &name) const override; - private: DDim GetDim(const std::string &name) const override; void SetDim(const std::string &name, const DDim &dim) override; @@ -327,6 +330,19 @@ void OpDescBind::InferShape(const BlockDescBind &block) const { PADDLE_ENFORCE(static_cast(infer_shape), "%s's infer_shape has not been registered", this->Type()); CompileTimeInferShapeContext ctx(*this, block); + if (VLOG_IS_ON(10)) { + std::ostringstream sout; + auto inames = this->InputArgumentNames(); + sout << " From ["; + std::copy(inames.begin(), inames.end(), + std::ostream_iterator(sout, ", ")); + sout << "] to ["; + auto onames = this->OutputArgumentNames(); + std::copy(onames.begin(), onames.end(), + std::ostream_iterator(sout, ", ")); + sout << "]"; + VLOG(10) << sout.str(); + } infer_shape(&ctx); } @@ -336,9 +352,13 @@ void OpDescBind::InferVarType(BlockDescBind *block) const { info.infer_var_type_(*this, block); } else { // all output type is LoDTensor by default + VLOG(10) << this->Type() + << " has not registered InferVarType. Set output variables to " + "LOD_TENSOR"; for (auto &out_pair : this->outputs_) { for (auto &out_var_name : out_pair.second) { - block->Var(out_var_name)->SetType(VarDesc::LOD_TENSOR); + block->FindRecursiveOrCreateVar(out_var_name) + ->SetType(VarDesc::LOD_TENSOR); } } } @@ -435,6 +455,12 @@ void CompileTimeInferShapeContext::SetDim(const std::string &name, const DDim &dim) { block_.FindVarRecursive(name)->SetShape(framework::vectorize(dim)); } +bool CompileTimeInferShapeContext::IsRuntime() const { return false; } + +VarDesc::VarType CompileTimeInferShapeContext::GetVarType( + const std::string &name) const { + return block_.FindVarRecursive(name)->GetType(); +} } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 2bb5e0e8ec29fb2df81549650aa0c65bc1e51c49..daade439e5232f06be72bc5bb1e2285124f2c3a4 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -92,8 +92,7 @@ struct OpKernelRegistrarFunctor { void operator()(const char* op_type) const { using T = typename KERNEL_TYPE::ELEMENT_TYPE; - OperatorWithKernel::OpKernelKey key(ToDataType(std::type_index(typeid(T))), - PlaceType()); + OpKernelType key(ToDataType(std::type_index(typeid(T))), PlaceType()); OperatorWithKernel::AllOpKernels()[op_type][key].reset(new KERNEL_TYPE); constexpr auto size = std::tuple_size>::value; diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 3be26fdc4fb6ebdd0ec427a2248b0f97d9edff01..3276f8af396fe58450a8dc6713fe61e49d5ca708 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -15,7 +15,9 @@ limitations under the License. */ #include "paddle/framework/operator.h" #include #include +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/shape_inference.h" +#include "paddle/framework/var_type.h" namespace paddle { namespace framework { @@ -126,7 +128,7 @@ OperatorBase::OperatorBase(const std::string& type, std::vector OperatorBase::InputVars() const { std::vector ret_val; - for (auto& o : outputs_) { + for (auto& o : inputs_) { ret_val.reserve(ret_val.size() + o.second.size()); ret_val.insert(ret_val.end(), o.second.begin(), o.second.end()); } @@ -252,8 +254,7 @@ std::vector ExecutionContext::MultiOutput( return res; } -std::ostream& operator<<(std::ostream& os, - const OperatorWithKernel::OpKernelKey& kernel_key) { +std::ostream& operator<<(std::ostream& os, const OpKernelType& kernel_key) { os << "place[" << kernel_key.place_ << "]:data_type[" << kernel_key.data_type_ << "]"; return os; @@ -365,7 +366,9 @@ class RuntimeInferShapeContext : public InferShapeContext { out_tensor->set_lod(in_tensor.lod()); } - private: + bool IsRuntime() const override { return true; } + + protected: DDim GetDim(const std::string& name) const override { Variable* var = scope_.FindVar(name); if (var->IsType()) { @@ -388,13 +391,31 @@ class RuntimeInferShapeContext : public InferShapeContext { } } + VarDesc::VarType GetVarType(const std::string& name) const override { + auto* var = scope_.FindVar(name); + return ToVarType(var->Type()); + } + + private: const OperatorBase& op_; const Scope& scope_; }; void OperatorWithKernel::Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const { - VLOG(3) << "Running operator " << this->Type(); + if (VLOG_IS_ON(1)) { + auto inputs = this->InputVars(); + auto outputs = this->OutputVars(true); + std::ostringstream sout; + sout << "Run operator " << this->Type() << " From ["; + std::ostream_iterator out_it(sout, ","); + std::copy(inputs.begin(), inputs.end(), out_it); + sout << "] to ["; + std::copy(outputs.begin(), outputs.end(), out_it); + sout << "]"; + VLOG(1) << sout.str(); + } + RuntimeInferShapeContext infer_shape_ctx(*this, scope); this->InferShape(&infer_shape_ctx); @@ -410,7 +431,7 @@ void OperatorWithKernel::Run(const Scope& scope, // check if op[type] have kernel for kernel_key OpKernelMap& kernels = kernels_iter->second; - auto kernel_key = OpKernelKey(IndicateDataType(ctx), dev_ctx); + auto kernel_key = GetKernelType(ctx); auto kernel_iter = kernels.find(kernel_key); if (kernel_iter == kernels.end()) { @@ -418,6 +439,41 @@ void OperatorWithKernel::Run(const Scope& scope, } kernel_iter->second->Compute(ctx); + + // throws errors if have. + dev_ctx.Finish(); +} +OpKernelType OperatorWithKernel::GetKernelType( + const ExecutionContext& ctx) const { + return OpKernelType(IndicateDataType(ctx), ctx.device_context()); +} +DataType OperatorWithKernel::IndicateDataType( + const ExecutionContext& ctx) const { + auto& scope = ctx.scope(); + int data_type = -1; + for (auto& input : this->inputs_) { + for (auto& ipt_name : input.second) { + auto* var = scope.FindVar(ipt_name); + if (var != nullptr) { + const Tensor* t = nullptr; + if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &(var->Get().value()); + } + if (t != nullptr) { + int tmp = static_cast(ToDataType(t->type())); + PADDLE_ENFORCE(tmp == data_type || data_type == -1, + "DataType of Paddle Op %s must be the same.", Type()); + data_type = tmp; + } + } + } + } + PADDLE_ENFORCE(data_type != -1, "DataType should be indicated by input"); + return static_cast(data_type); } } // namespace framework diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index b8a7040ed024fc7b19980beef3d8b367dfdd7f50..60861d92933dd100f877bec8d43f9b924f951e60 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -298,11 +298,10 @@ class ExecutionContext { } #ifdef PADDLE_WITH_CUDA - const platform::CUDADeviceContext& cuda_device_context() const { + const inline platform::CUDADeviceContext& cuda_device_context() const { PADDLE_ENFORCE(platform::is_gpu_place(device_context_.GetPlace())); - auto cuda_ctx = - reinterpret_cast(&device_context_); - return *cuda_ctx; + return *reinterpret_cast( + &device_context_); } #endif @@ -346,27 +345,10 @@ class OpKernel : public OpKernelBase { using ELEMENT_TYPE = T; }; -class OperatorWithKernel : public OperatorBase { - public: - struct OpKernelKey { - platform::Place place_; - DataType data_type_; - - OpKernelKey(DataType data_type, platform::Place place) - : place_(place), data_type_(data_type) {} - - OpKernelKey(DataType data_type, const platform::DeviceContext& dev_ctx) - : place_(dev_ctx.GetPlace()), data_type_(data_type) {} - - bool operator==(const OpKernelKey& o) const { - return platform::places_are_same_class(place_, o.place_) && - data_type_ == o.data_type_; - } - }; - - struct OpKernelHash { +struct OpKernelType { + struct Hash { std::hash hash_; - size_t operator()(const OpKernelKey& key) const { + size_t operator()(const OpKernelType& key) const { int place = key.place_.which(); int data_type = static_cast(key.data_type_); int pre_hash = data_type << NUM_PLACE_TYPE_LIMIT_IN_BIT | @@ -375,9 +357,26 @@ class OperatorWithKernel : public OperatorBase { } }; + platform::Place place_; + DataType data_type_; + + OpKernelType(DataType data_type, platform::Place place) + : place_(place), data_type_(data_type) {} + + OpKernelType(DataType data_type, const platform::DeviceContext& dev_ctx) + : place_(dev_ctx.GetPlace()), data_type_(data_type) {} + + bool operator==(const OpKernelType& o) const { + return platform::places_are_same_class(place_, o.place_) && + data_type_ == o.data_type_; + } +}; + +class OperatorWithKernel : public OperatorBase { + public: using OpKernelMap = - std::unordered_map, - OpKernelHash>; + std::unordered_map, + OpKernelType::Hash>; OperatorWithKernel(const std::string& type, const VariableNameMap& inputs, const VariableNameMap& outputs, const AttributeMap& attrs) @@ -405,42 +404,15 @@ class OperatorWithKernel : public OperatorBase { } protected: + virtual OpKernelType GetKernelType(const ExecutionContext& ctx) const; + + private: // indicate kernel DataType by input data. Defaultly all input data must be // same. - virtual DataType IndicateDataType(const ExecutionContext& ctx) const { - VLOG(3) << "Default IndicateDataType " << this->Type(); - auto& scope = ctx.scope(); - int data_type = -1; - for (auto& input : this->inputs_) { - for (auto& ipt_name : input.second) { - auto* var = scope.FindVar(ipt_name); - if (var != nullptr) { - const Tensor* t = nullptr; - if (var->IsType()) { - t = &var->Get(); - } else if (var->IsType()) { - t = &var->Get(); - } else if (var->IsType()) { - t = &(var->Get().value()); - } - if (t != nullptr) { - int tmp = static_cast(ToDataType(t->type())); - VLOG(3) << "Input " << ipt_name << " with data_type " << tmp; - PADDLE_ENFORCE(tmp == data_type || data_type == -1, - "DataType of Paddle Op %s must be the same.", - Type()); - data_type = tmp; - } - } - } - } - PADDLE_ENFORCE(data_type != -1, "DataType should be indicated by input"); - return static_cast(data_type); - } + DataType IndicateDataType(const ExecutionContext& ctx) const; }; -std::ostream& operator<<(std::ostream& os, - const OperatorWithKernel::OpKernelKey& kernel_key); +std::ostream& operator<<(std::ostream& os, const OpKernelType& kernel_key); extern bool OpSupportGPU(const std::string& op_type); diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 42e0d52eed3911d8e684e76a88bc690ca0783ce5..1e19f82b341768142258ba4a5dfa246d87ba4c43 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -114,8 +114,8 @@ class OpWithKernelTest : public OperatorWithKernel { protected: void InferShape(framework::InferShapeContext* ctx) const override {} - DataType IndicateDataType(const ExecutionContext& ctx) const override { - return DataType::FP32; + OpKernelType GetKernelType(const ExecutionContext& ctx) const override { + return OpKernelType(DataType::FP32, ctx.device_context()); } }; diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc index 14cc530448379eb6d4bf0435f607494aa01ef5b5..9428b8a07ea0af005f6e960ddaa02da624ad9d97 100644 --- a/paddle/framework/scope.cc +++ b/paddle/framework/scope.cc @@ -47,8 +47,12 @@ Variable* Scope::Var(const std::string& name) { return v; } -Variable* Scope::Var() { - return Var(string::Sprintf("%p.%d", this, vars_.size())); +Variable* Scope::Var(std::string* name) { + auto var_name = string::Sprintf("%p.%d", this, vars_.size()); + if (name != nullptr) { + *name = var_name; + } + return Var(var_name); } Variable* Scope::FindVar(const std::string& name) const { @@ -94,5 +98,23 @@ void Scope::DeleteScope(Scope* scope) { delete scope; } +void Scope::Rename(const std::string& origin_name, + const std::string& new_name) const { + auto origin_it = vars_.find(origin_name); + PADDLE_ENFORCE(origin_it != vars_.end(), + "Cannot find original variable with name %s", origin_name); + auto new_it = vars_.find(new_name); + PADDLE_ENFORCE(new_it == vars_.end(), + "The variable with name %s is already in the scope", new_name); + vars_[new_name] = origin_it->second; + vars_.erase(origin_it); +} + +std::string Scope::Rename(const std::string& origin_name) const { + auto var_name = string::Sprintf("%p.%d", this, vars_.size()); + Rename(origin_name, var_name); + return var_name; +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index ac334da5ef0c8ad563b6be5413df33f5d0bdbcf8..c2aafb6ad825f9bd9ffef754923a15afdeaa8e5c 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -49,7 +49,7 @@ class Scope { Variable* Var(const std::string& name); /// Create a variable with a scope-unique name. - Variable* Var(); + Variable* Var(std::string* name = nullptr); /// Find a variable in the scope or any of its ancestors. Returns /// nullptr if cannot find. @@ -68,11 +68,18 @@ class Scope { // enumerate all the variables current contains. std::vector GetAllNames(bool recursive = false) const; + // Rename variable to a new name + void Rename(const std::string& origin_name, + const std::string& new_name) const; + + // Rename variable to a new name and return the new name + std::string Rename(const std::string& origin_name) const; + private: // Call Scope::NewScope for a sub-scope. explicit Scope(Scope const* parent) : parent_(parent) {} - std::unordered_map vars_; + mutable std::unordered_map vars_; mutable std::list kids_; Scope const* parent_{nullptr}; diff --git a/paddle/framework/shape_inference.cc b/paddle/framework/shape_inference.cc index 8169df8e4629e2d02d3dabcd6a8a102ad0077a81..0af41b164f5894db17b2f86d4eba371cf05e3b41 100644 --- a/paddle/framework/shape_inference.cc +++ b/paddle/framework/shape_inference.cc @@ -46,6 +46,23 @@ void InferShapeContext::SetDims(const std::vector &names, SetDim(names[i], dims[i]); } } +std::vector InferShapeContext::GetInputsVarType( + const std::string &name) const { + return GetVarTypes(Inputs(name)); +} +std::vector InferShapeContext::GetOutputsVarType( + const std::string &name) const { + return GetVarTypes(Outputs(name)); +} +std::vector InferShapeContext::GetVarTypes( + const std::vector &names) const { + std::vector retv; + retv.resize(names.size()); + std::transform(names.begin(), names.end(), retv.begin(), + std::bind(std::mem_fn(&InferShapeContext::GetVarType), this, + std::placeholders::_1)); + return retv; +} } // namespace framework } // namespace paddle diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index 6f19900ef1a3e88fe78d457a03c344ea586ab551..7d36ead2ca85328c7843b3b5d423cf8e921d1c93 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/framework/attribute.h" #include "paddle/framework/ddim.h" +#include "paddle/framework/framework.pb.h" namespace paddle { namespace framework { @@ -26,6 +27,10 @@ class InferShapeContext { virtual bool HasInput(const std::string &name) const = 0; virtual bool HasOutput(const std::string &name) const = 0; + std::vector GetInputsVarType(const std::string &name) const; + std::vector GetOutputsVarType( + const std::string &name) const; + virtual bool HasInputs(const std::string &name) const = 0; virtual bool HasOutputs(const std::string &name) const = 0; @@ -46,6 +51,8 @@ class InferShapeContext { virtual void ShareLoD(const std::string &in, const std::string &out, size_t i = 0, size_t j = 0) const = 0; + virtual bool IsRuntime() const = 0; + protected: virtual framework::DDim GetDim(const std::string &name) const = 0; virtual void SetDim(const std::string &name, const framework::DDim &dim) = 0; @@ -55,6 +62,11 @@ class InferShapeContext { void SetDims(const std::vector &names, const std::vector &dims); + + std::vector GetVarTypes( + const std::vector &names) const; + + virtual VarDesc::VarType GetVarType(const std::string &name) const = 0; }; } // namespace framework diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h index 9eab67561a42b3fb4e22d8475ad5eeb146a72f1c..28d0fcf94ec31c82476e093f93ccee222a0c9d9a 100644 --- a/paddle/framework/tensor.h +++ b/paddle/framework/tensor.h @@ -125,7 +125,7 @@ class Tensor { * @param[in] end_idx The index of the end row(exclusive) to slice. * The index number begins from 0. */ - inline Tensor Slice(const int& begin_idx, const int& end_idx) const; + inline Tensor Slice(int begin_idx, int end_idx) const; platform::Place place() const { PADDLE_ENFORCE_NOT_NULL( diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index bcccdd5881775e199297dce7e70aaf6aae62d95a..7e88e039611007d17156d10f852eb46f3ee8e7a3 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -52,7 +52,7 @@ struct SizeOfTypeFunctor { }; static inline size_t SizeOfType(std::type_index type) { - SizeOfTypeFunctor functor; + SizeOfTypeFunctor functor; size_t size = functor(type); PADDLE_ENFORCE(size != 0UL, "Cannot get size of type %s", type.name()); return size; @@ -228,7 +228,7 @@ inline void Tensor::CopyFromVector(const std::vector& src, #endif } -inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const { +inline Tensor Tensor::Slice(int begin_idx, int end_idx) const { check_memory_size(); PADDLE_ENFORCE_GE(begin_idx, 0, "The start row index must be greater than 0."); diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index afeeb1914ac30188b93c3b9da30bb5ceaf74416e..baeb98c9bd49ec65da5931bcbe33ab788f86f3e8 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -29,6 +29,7 @@ class OpDescBind; class BlockDescBind; class BlockDesc; class InferShapeContext; +class BlockDescBind; using VariableNameMap = std::map>; @@ -46,7 +47,8 @@ using OpCreator = std::function>( const OpDescBind&, const std::unordered_set& /*no_grad_set*/, - std::unordered_map* /*grad_to_var*/)>; + std::unordered_map* /*grad_to_var*/, + const std::vector& grad_block)>; using InferVarTypeFN = std::function; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 8e92c81d1137472737230be79d71824593d3256f..0babec29f6f4412ed29deeafe24470e86b30a636 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -37,13 +37,29 @@ std::vector VarDescBind::Shape() const { DataType VarDescBind::GetDataType() const { return tensor_desc().data_type(); } void VarDescBind::SetLoDLevel(int32_t lod_level) { - PADDLE_ENFORCE(desc_.type() == VarDesc::LOD_TENSOR); - desc_.mutable_lod_tensor()->set_lod_level(lod_level); + switch (desc_.type()) { + case VarDesc::LOD_TENSOR: + desc_.mutable_lod_tensor()->set_lod_level(lod_level); + break; + case VarDesc::LOD_TENSOR_ARRAY: + desc_.mutable_tensor_array()->set_lod_level(lod_level); + break; + default: + PADDLE_THROW("Tensor type=%d does not support LoDLevel", + desc_.tensor_array().lod_level()); + } } int32_t VarDescBind::GetLodLevel() const { - PADDLE_ENFORCE(desc_.type() == VarDesc::LOD_TENSOR); - return desc_.lod_tensor().lod_level(); + switch (desc_.type()) { + case VarDesc::LOD_TENSOR: + return desc_.lod_tensor().lod_level(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.tensor_array().lod_level(); + default: + PADDLE_THROW("Tensor type=%d does not support LoDLevel", + desc_.tensor_array().lod_level()); + } } const TensorDesc &VarDescBind::tensor_desc() const { @@ -53,6 +69,8 @@ const TensorDesc &VarDescBind::tensor_desc() const { return desc_.selected_rows(); case VarDesc::LOD_TENSOR: return desc_.lod_tensor().tensor(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.tensor_array().tensor(); default: PADDLE_THROW("Unexpected branch."); } @@ -66,6 +84,8 @@ TensorDesc *VarDescBind::mutable_tensor_desc() { return desc_.mutable_selected_rows(); case VarDesc::LOD_TENSOR: return desc_.mutable_lod_tensor()->mutable_tensor(); + case VarDesc::LOD_TENSOR_ARRAY: + return desc_.mutable_tensor_array()->mutable_tensor(); default: PADDLE_THROW("Unexpected branch."); } diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h index 70daa20e8d99abc5759655adf538a8c197e9ec6a..5cf4608944c5011d798fbde060002a57be8f6102 100644 --- a/paddle/framework/var_desc.h +++ b/paddle/framework/var_desc.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include "glog/logging.h" #include "paddle/framework/framework.pb.h" namespace paddle { diff --git a/paddle/framework/var_type.h b/paddle/framework/var_type.h new file mode 100644 index 0000000000000000000000000000000000000000..0f19870bec3e69d07278507cc556a86bbd25d12d --- /dev/null +++ b/paddle/framework/var_type.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/framework.pb.h" +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/lod_tensor.h" +#include "paddle/framework/lod_tensor_array.h" + +namespace paddle { +namespace framework { +inline VarDesc::VarType ToVarType(std::type_index type) { + if (type.hash_code() == typeid(LoDTensor).hash_code()) { + return VarDesc_VarType_LOD_TENSOR; + } else if (type.hash_code() == typeid(LoDRankTable).hash_code()) { + return VarDesc_VarType_LOD_RANK_TABLE; + } else if (type.hash_code() == typeid(LoDTensorArray).hash_code()) { + return VarDesc_VarType_LOD_TENSOR_ARRAY; + } else if (type.hash_code() == typeid(SelectedRows).hash_code()) { + return VarDesc_VarType_SELECTED_ROWS; + } else { + PADDLE_THROW("ToVarType:Unsupported type %s", type.name()); + } +} + +template +inline void VisitVarType(const Variable& var, Visitor visitor) { + switch (ToVarType(var.Type())) { + case VarDesc_VarType_LOD_TENSOR: + visitor(var.Get()); + return; + case VarDesc_VarType_LOD_RANK_TABLE: + visitor(var.Get()); + return; + case VarDesc_VarType_LOD_TENSOR_ARRAY: + visitor(var.Get()); + return; + case VarDesc_VarType_SELECTED_ROWS: + visitor(var.Get()); + return; + default: + PADDLE_THROW("Not supported visit type, %d", ToVarType(var.Type())); + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/variable.h b/paddle/framework/variable.h index cde5ec2413ad01a0396e19fa617688af0eafbc75..e5a94759f9230ab4ce9d2cc24849a2debb8a5e2f 100644 --- a/paddle/framework/variable.h +++ b/paddle/framework/variable.h @@ -48,6 +48,11 @@ class Variable { void Clear() { holder_.reset(); } + std::type_index Type() const { + PADDLE_ENFORCE(holder_ != nullptr, "Must hold memory"); + return holder_->Type(); + } + private: struct Placeholder { virtual ~Placeholder() {} diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 4fd72d64a90ae6f16dd1499ceb7fba6e40fe4cea..9b2779b42cad324253dadf27dbff20fd8e8c8e16 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -45,6 +45,7 @@ if(WITH_GPU) add_simple_unittest(BlockExpandOpTest) add_simple_unittest(CropOpTest) add_simple_unittest(SwitchOpTest) + add_simple_unittest(ScaleSubRegionOpTest) endif() add_simple_unittest(Im2ColTest) diff --git a/paddle/function/FunctionTest.h b/paddle/function/FunctionTest.h index ba446bf92da264fafa1fb47a2c30da9cb13176ce..370940532ef40335be54a3e6467de0409e923ec4 100644 --- a/paddle/function/FunctionTest.h +++ b/paddle/function/FunctionTest.h @@ -110,6 +110,7 @@ public: function2_(FunctionBase::funcRegistrar_.createByType(name2)) { function1_->init(config); function2_->init(config); + initArgsCallback_ = nullptr; } ~Compare2Function() {} @@ -170,6 +171,10 @@ public: *seq2_)); } + void registerInitCallback(std::function callback) { + initArgsCallback_ = callback; + } + // output need only contains shape, do not contains data. void addOutputs(const BufferArg& output, ArgType argType = ASSIGN_TO) { size_t size = @@ -340,6 +345,10 @@ protected: initArg(*func1Inputs_[i]); } + if (initArgsCallback_ != nullptr) { + initArgsCallback_(*func1Inputs_[i], i); + } + copyArg_(*func1Inputs_[i], *func2Inputs_[i]); } } @@ -386,6 +395,7 @@ protected: std::shared_ptr seq1_; std::shared_ptr seq2_; test::CopyArgument copyArg_; + std::function initArgsCallback_; }; class CpuGpuFuncCompare diff --git a/paddle/function/ScaleSubRegionOp.cpp b/paddle/function/ScaleSubRegionOp.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a080505d7df83a6c0a9d88fbcb7863fc0e1f7b21 --- /dev/null +++ b/paddle/function/ScaleSubRegionOp.cpp @@ -0,0 +1,155 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionOp.h" +#include "paddle/function/TensorShape.h" + +namespace paddle { + +template <> +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + memcpy(outputs, inputs, number * channel * height * width * sizeof(real)); + + for (int n = 0; n < number; ++n) { + // indices start from 1 + int offset = n * 6; + for (int c = indices[offset] - 1; c < indices[offset + 1]; ++c) { + for (int h = indices[offset + 2] - 1; h < indices[offset + 3]; ++h) { + for (int w = indices[offset + 4] - 1; w < indices[offset + 5]; ++w) { + int idx = ((n * channel + c) * height + h) * width + w; + outputs[idx] *= value; + } + } + } + } +} + +template <> +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + for (int n = 0; n < number; ++n) { + for (int c = 0; c < channel; ++c) { + for (int h = 0; h < height; ++h) { + for (int w = 0; w < width; ++w) { + int idx = ((n * channel + c) * height + h) * width + w; + int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && + h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && + w <= (indices[offset + 5] - 1)) { + outGrad[idx] += inGrad[idx] * value; + } else { + outGrad[idx] += inGrad[idx]; + } + } + } + } + } +} + +/** + * \brief For each instance, ScaleSubRegion can be used to multiply a value to + * a specified sub continuous region. By providing start index and end + * index for C/H/W, you can specify the location and shape of the region. + * + * Argument in this Function: + * \param inputs A 4-D tensor with shape [N, C, H, W], only one input. + * \param indices A 2-D tensor with shape [N, 6], indicates the sub region. + * \param outputs A 4-D tensor with same shape as inputs, output value. + */ +template +class ScaleSubRegionFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { conf_ = config; } + + void calc(const BufferArgs& inputs, const BufferArgs& outputs) override { + CHECK_EQ(2UL, inputs.size()); + CHECK_EQ(1UL, outputs.size()); + CHECK_EQ(outputs[0].getArgType(), ASSIGN_TO); + + TensorShape shape = inputs[0].shape(); + + ScaleSubRegion(outputs[0].data(), + inputs[0].data(), + inputs[1].data(), + shape, + conf_); + } + +private: + FuncConfig conf_; +}; + +/** + * \brief The backward propagation of ScaleSubRegion Function. + * + * Argument in this Function: + * \param inputs A 4-D tensor with shape [N, C, H, W], output gradient. + * \param indices A 2-D tensor with shape [N, 6], indicates the sub region. + * \param outputs A 4-D tensor with shape [N, C, H, W], gradient of input value. + */ + +template +class ScaleSubRegionGradFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { conf_ = config; } + + void calc(const BufferArgs& inputs, const BufferArgs& outputs) override { + CHECK_EQ(2UL, inputs.size()); + CHECK_EQ(1UL, outputs.size()); + CHECK_EQ(outputs[0].getArgType(), ADD_TO); + + TensorShape shape = inputs[0].shape(); + + ScaleSubRegionGrad(inputs[0].data(), + outputs[0].data(), + inputs[1].data(), + shape, + conf_); + } + +private: + FuncConfig conf_; +}; + +REGISTER_TYPED_FUNC(ScaleSubRegion, CPU, ScaleSubRegionFunc); +REGISTER_TYPED_FUNC(ScaleSubRegionGrad, CPU, ScaleSubRegionGradFunc); +#ifdef PADDLE_WITH_CUDA +REGISTER_TYPED_FUNC(ScaleSubRegion, GPU, ScaleSubRegionFunc); +REGISTER_TYPED_FUNC(ScaleSubRegionGrad, GPU, ScaleSubRegionGradFunc); +#endif + +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOp.h b/paddle/function/ScaleSubRegionOp.h new file mode 100644 index 0000000000000000000000000000000000000000..0480c8577f3fbf3bc9e94b635df96a31b103e9e3 --- /dev/null +++ b/paddle/function/ScaleSubRegionOp.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Function.h" + +namespace paddle { + +/** + * \brief Function to multiply a value to values in specified sub continuous + * region. Indices must be provided to indcate the location and shape of + * the region and the multiplied value is passed by configure variable. + * + * + * \param[out] outputs Output value. + * \param[in] inputs Input data which contains NCHW information. + * \param[in] indices Indices data to indcate the sub region. + * \param[in] shape Tensor shape of input value. + * \param[in] conf Configure variable which contains the multiplied value. + */ +template +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf); + +/** + * \brief Backward propagation function of ScaleSubRegion. + * + * \param[out] inGrad Gradients of previous layer. + * \param[in] outGrad Output gradient. + * \param[in] indices Indices data. + * \param[in] shape The Shape of input tensor. + * \param[in] conf Configure variable. + */ +template +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf); +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOpGpu.cu b/paddle/function/ScaleSubRegionOpGpu.cu new file mode 100644 index 0000000000000000000000000000000000000000..8aae2e44c3fdc8b516e66ecfd2e04f466a17dde9 --- /dev/null +++ b/paddle/function/ScaleSubRegionOpGpu.cu @@ -0,0 +1,116 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionOp.h" +#include "hl_base.h" + +namespace paddle { + +__global__ void KeScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + real value, + int channel, + int height, + int width, + int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % width; + const int h = (idx / width) % height; + const int c = (idx / width / height) % channel; + const int n = idx / width / height / channel; + + const int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && w <= (indices[offset + 5] - 1)) { + outputs[idx] = inputs[idx] * value; + } else { + outputs[idx] = inputs[idx]; + } + } +} + +template <> +void ScaleSubRegion(real* outputs, + const real* inputs, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + size_t nth = number * channel * height * width; + int blockSize = 1024; + int gridSize = (nth + blockSize - 1) / blockSize; + + KeScaleSubRegion<<>>( + outputs, inputs, indices, value, channel, height, width, nth); + CHECK_SYNC("ScaleSubRegion"); +} + +__global__ void KeScaleSubRegionDiff(const real* inGrad, + real* outGrad, + const real* indices, + real value, + int channel, + int height, + int width, + int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % width; + const int h = (idx / width) % height; + const int c = (idx / width / height) % channel; + const int n = idx / width / height / channel; + + const int offset = n * 6; + if (c >= (indices[offset] - 1) && c <= (indices[offset + 1] - 1) && + h >= (indices[offset + 2] - 1) && h <= (indices[offset + 3] - 1) && + w >= (indices[offset + 4] - 1) && w <= (indices[offset + 5] - 1)) { + outGrad[idx] += inGrad[idx] * value; + } else { + outGrad[idx] += inGrad[idx]; + } + } +} + +template <> +void ScaleSubRegionGrad(const real* inGrad, + real* outGrad, + const real* indices, + const TensorShape shape, + const FuncConfig& conf) { + real value = conf.get("value"); + + int number = shape[0]; + int channel = shape[1]; + int height = shape[2]; + int width = shape[3]; + + size_t nth = number * channel * height * width; + int blockSize = 1024; + int gridSize = (nth + blockSize - 1) / blockSize; + + KeScaleSubRegionDiff<<>>( + inGrad, outGrad, indices, value, channel, height, width, nth); + CHECK_SYNC("ScaleSubRegionGrad"); +} + +} // namespace paddle diff --git a/paddle/function/ScaleSubRegionOpTest.cpp b/paddle/function/ScaleSubRegionOpTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..43331f258dddaa43cbc8cc77519e299de7e98290 --- /dev/null +++ b/paddle/function/ScaleSubRegionOpTest.cpp @@ -0,0 +1,72 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "FunctionTest.h" + +namespace paddle { + +TEST(ScaleSubRegion, real) { + for (size_t numSamples : {5, 32}) { + for (size_t channels : {5, 32}) { + for (size_t imgSizeH : {5, 33}) { + for (size_t imgSizeW : {5, 32}) { + for (real value : {-0.5, 0.0, 0.5}) { + for (bool firstHalf : {false, true}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW; + + for (bool testGrad : {false, true}) { + CpuGpuFuncCompare compare( + testGrad ? "ScaleSubRegionGrad" : "ScaleSubRegion", + FuncConfig().set("value", value)); + + TensorShape shape{numSamples, channels, imgSizeH, imgSizeW}; + TensorShape indicesShape{numSamples, 6}; + + compare.addInputs(BufferArg(VALUE_TYPE_FLOAT, shape)); + compare.addInputs(BufferArg(VALUE_TYPE_FLOAT, indicesShape)); + + compare.registerInitCallback([=](BufferArg& arg, size_t index) { + if (index == 1) { + real* data = (real*)arg.data(); + + for (size_t i = 0; i < numSamples; ++i) { + size_t offset = i * 6; + data[offset] = firstHalf ? 1 : channels / 2; + data[offset + 1] = firstHalf ? channels / 2 : channels; + data[offset + 2] = firstHalf ? 1 : imgSizeH / 2; + data[offset + 3] = firstHalf ? imgSizeH / 2 : imgSizeH; + data[offset + 4] = firstHalf ? 1 : imgSizeW / 2; + data[offset + 5] = firstHalf ? imgSizeW / 2 : imgSizeW; + } + } + }); + + compare.addOutputs( + BufferArg( + VALUE_TYPE_FLOAT, shape, testGrad ? ADD_TO : ASSIGN_TO), + testGrad ? ADD_TO : ASSIGN_TO); + compare.run(); + } + } + } + } + } + } + } +} + +} // namespace paddle diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index 9db6d252d97bfeee3fe376bcda431fe94c65a678..8e66b1f0db5d8a365a5aa9b98d2fb3f867458411 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -395,14 +395,24 @@ real AucEvaluator::evalImp(std::vector& arguments) { CHECK_LE(arguments.size(), (size_t)3); MatrixPtr output = arguments[0].value; IVectorPtr label = arguments[1].ids; + MatrixPtr labelval = arguments[1].value; bool supportWeight = (3 == arguments.size()) ? true : false; MatrixPtr weight = supportWeight ? arguments[2].value : nullptr; - if (nullptr == output || nullptr == label || - (supportWeight && nullptr == weight)) { + + if (nullptr == output || (supportWeight && nullptr == weight)) { return 0; } size_t insNum = output->getHeight(); size_t outputDim = output->getWidth(); + // Copy label from value to a vector. + if (nullptr == label && nullptr != labelval) { + // label width is 1 + CHECK_EQ(1U, labelval->getWidth()); + VectorPtr vec = + Vector::create(labelval->getData(), insNum, output->useGpu()); + label = vec->castToInt(); + } + CHECK_EQ(insNum, label->getSize()); if (supportWeight) { CHECK_EQ(insNum, weight->getHeight()); @@ -443,6 +453,7 @@ real AucEvaluator::evalImp(std::vector& arguments) { int* labelD = label->getData(); real* weightD = supportWeight ? weight->getData() : nullptr; size_t pos = realColumnIdx_; + for (size_t i = 0; i < insNum; ++i) { real value = outputD[pos]; uint32_t binIdx = static_cast(value * kBinNum_); diff --git a/paddle/gserver/layers/ConvBaseProjection.cpp b/paddle/gserver/layers/ConvBaseProjection.cpp index 08f36c516cfdadd42e9333c1c5a7a247df1f263e..19efed7b52ee07a5c509d069c286ccc3b21602f4 100644 --- a/paddle/gserver/layers/ConvBaseProjection.cpp +++ b/paddle/gserver/layers/ConvBaseProjection.cpp @@ -17,7 +17,7 @@ limitations under the License. */ namespace paddle { -ThreadLocalD> ConvBaseProjection::convMem_; +ThreadLocalD> ConvBaseProjection::convMem_; ConvBaseProjection::ConvBaseProjection(const ProjectionConfig &config, ParameterPtr parameter, @@ -175,18 +175,18 @@ void ConvBaseProjection::reshape(int batchSize) { } void *ConvBaseProjection::getSpaceBytes(size_t size) { - std::vector &convMem = *convMem_; + std::vector &convMem = *convMem_; if (convMem.empty()) { int numDevices = hl_get_device_count(); convMem.resize(numDevices); } int devId = hl_get_device(); - MemoryHandle **localMem = &(convMem[devId]); - if (NULL == *localMem || size > (*localMem)->getAllocSize()) { - *localMem = new GpuMemoryHandle(size); + MemoryHandlePtr localMem = convMem[devId]; + if (NULL == localMem || size > localMem->getAllocSize()) { + localMem = std::make_shared(size); } - return (*localMem)->getBuf(); + return localMem->getBuf(); } ConvBaseProjection::~ConvBaseProjection() { diff --git a/paddle/gserver/layers/ConvBaseProjection.h b/paddle/gserver/layers/ConvBaseProjection.h index ebdb57845bb36ac607b1e4c8e02f9d20b6e82a36..bb7ffa627b745f45b0f210cdb58ef87d6990af73 100644 --- a/paddle/gserver/layers/ConvBaseProjection.h +++ b/paddle/gserver/layers/ConvBaseProjection.h @@ -105,7 +105,7 @@ protected: bool bias_; std::unique_ptr weight_; - static ThreadLocalD> convMem_; + static ThreadLocalD> convMem_; }; } // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.cpp b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0f2b67fd758ec1513f42c4cb1a36f2f3915f4740 --- /dev/null +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.cpp @@ -0,0 +1,222 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MKLDNNAddtoLayer.h" + +using namespace mkldnn; // NOLINT + +namespace paddle { + +REGISTER_LAYER(mkldnn_addto, MKLDNNAddtoLayer); + +bool MKLDNNAddtoLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + if (!MKLDNNLayer::init(layerMap, parameterMap)) { + return false; + } + + layerSize_ = getSize(); + for (size_t i = 0; i < inputLayers_.size(); i++) { + CHECK_EQ(layerSize_, inputLayers_[i]->getSize()) << "input size must equal"; + } + if (biasParameter_.get() != NULL) { + biases_ = + std::unique_ptr(new Weight(1, layerSize_, biasParameter_, 0)); + } + return true; +} + +void MKLDNNAddtoLayer::reshape( + int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) { + CHECK_EQ(layerSize_, getSize()) << "this layer size can not be changed"; + reshapeInput(bs, ih, iw); + ic = inputLayers_[0]->getSize() / ih / iw; + CHECK_EQ((size_t)ic * ih * iw, inputLayers_[0]->getSize()); + CHECK_EQ(inputElemenCnt_, (size_t)bs * ic * ih * iw); + for (size_t i = 0; i < inputLayers_.size(); i++) { + CHECK_EQ(int64_t(bs), inputLayers_[i]->getOutput().getBatchSize()); + CHECK_EQ(layerSize_, inputLayers_[i]->getSize()); + } + + oc = ic; + oh = ih; + ow = iw; + reshapeOutput(oh, ow); + resizeOutput(bs, oc * oh * ow); +} + +void MKLDNNAddtoLayer::resetFwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + resetFwdBuffers(inVals_, bias, out); + in = inVals_[0]; + + std::shared_ptr fwdPD; + std::shared_ptr biasPD; + resetFwdPD(fwdPD, biasPD, inVals_, bias, out); + + resetFwdPipeline(pipeline, fwdPD, biasPD, inVals_, bias, out); +} + +void MKLDNNAddtoLayer::resetBwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + resetBwdBuffers(inGrads_, bias, out); + in = inGrads_[0]; + + // backward only need share output grad to input grad + for (size_t i = 0; i < inGrads_.size(); i++) { + if (inGrads_[i] != nullptr) { + inGrads_[i] = out; + inputLayers_[i]->getOutputGrad()->setData(inGrads_[i]->getData()); + } + } + + // backward bias + bwdBias_ = nullptr; + if (bias) { + std::vector scales(bs_, 1.0); + std::vector srcPDs(bs_, bias->getPrimitiveDesc()); + auto biasPD = sum::primitive_desc(bias->getMemoryDesc(), scales, srcPDs); + std::vector srcs; + for (size_t i = 0; i < grads_.size(); ++i) { + srcs.push_back(*(grads_[i])); + } + bwdBias_.reset(new sum(biasPD, srcs, *bias)); + pipeline.push_back(*bwdBias_); + } +} + +void MKLDNNAddtoLayer::updateWeights(const UpdateCallback& callback) { + if (biases_ && biases_->getWGrad()) { + biases_->getParameterPtr()->incUpdate(callback); + } +} + +void MKLDNNAddtoLayer::prepareBias(MKLDNNMatrixPtr& bias, + const MatrixPtr& biasMat, + const MKLDNNMatrixPtr& out, + std::vector& outs) { + auto pd = MKLDNNMatrix::createPrimitiveDesc( + {(int)layerSize_}, memory::format::x, engine_); + bias = MKLDNNMatrix::create(pd, biasMat); + outs.clear(); + real* data = out->getData(); + CHECK_EQ(bs_ * layerSize_, out->getElementCnt()); + for (int i = 0; i < bs_; ++i) { + MatrixPtr tmp = + Matrix::create(data + i * layerSize_, 1, layerSize_, false, false); + outs.push_back(MKLDNNMatrix::create(bias->getPrimitiveDesc(), tmp)); + } +} + +void MKLDNNAddtoLayer::resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + inputs.resize(inputLayers_.size()); + for (size_t i = 0; i < inputs.size(); i++) { + resetInValue(inputs[i], nullptr, i); + CHECK(inputs[i]); + inputs[i]->downSpatial(); + } + for (size_t i = 1; i < inputs.size(); i++) { + CHECK_PRIMITIVE_DESC_EQ(inputs[i], inputs[0]->getPrimitiveDesc()); + } + + resetOutValue(out, inputs[0]->getPrimitiveDesc()); + + if (biases_ && biases_->getW()) { + prepareBias(bias, biases_->getW(), out, vals_); + } else { + bias = nullptr; + } +} + +void MKLDNNAddtoLayer::resetFwdPD(std::shared_ptr& pd, + std::shared_ptr& biasPD, + std::vector& inputs, + MKLDNNMatrixPtr bias, + MKLDNNMatrixPtr out) { + std::vector scales(inputs.size(), 1.0); + std::vector srcPDs; + for (size_t i = 0; i < inputs.size(); i++) { + srcPDs.push_back(inputs[i]->getPrimitiveDesc()); + } + CHECK(out); + pd.reset(new sum::primitive_desc(out->getMemoryDesc(), scales, srcPDs)); + CHECK_PRIMITIVE_DESC_EQ(out, pd->dst_primitive_desc()); + + biasPD = nullptr; + if (bias) { + std::vector scales(2, 1.0); + std::vector srcPDs(2, bias->getPrimitiveDesc()); + biasPD.reset( + new sum::primitive_desc(bias->getMemoryDesc(), scales, srcPDs)); + CHECK_PRIMITIVE_DESC_EQ(bias, biasPD->dst_primitive_desc()); + } +} + +void MKLDNNAddtoLayer::resetFwdPipeline( + std::vector& pipeline, + std::shared_ptr& pd, + std::shared_ptr& biasPD, + std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + std::vector srcs; + for (size_t i = 0; i < inputs.size(); i++) { + srcs.push_back(*(inputs[i])); + } + fwd_.reset(new sum(*pd, srcs, *out)); + pipeline.push_back(*fwd_); + + fwdBias_.clear(); + if (biasPD == nullptr || bias == nullptr) { + return; + } + fwdBias_.resize(vals_.size()); + for (size_t i = 0; i < vals_.size(); ++i) { + std::vector srcs; + srcs.push_back(*(vals_[i])); + srcs.push_back(*bias); + fwdBias_[i].reset(new sum(*biasPD, srcs, *vals_[i])); + pipeline.push_back(*fwdBias_[i]); + } +} + +void MKLDNNAddtoLayer::resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) { + CHECK(outVal_); + resetOutGrad(out, outVal_->getPrimitiveDesc()); + CHECK(out); + + inputs.resize(inputLayers_.size()); + for (size_t i = 0; i < inputs.size(); i++) { + resetInGrad(inputs[i], inVal_->getPrimitiveDesc(), i); + CHECK_PRIMITIVE_DESC_EQ(inputs[i], out->getPrimitiveDesc()); + } + + if (biases_ && biases_->getWGrad()) { + prepareBias(bias, biases_->getWGrad(), out, grads_); + } else { + bias = nullptr; + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNAddtoLayer.h b/paddle/gserver/layers/MKLDNNAddtoLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..24504b7b4f50726e2b2757ca3029461cdc27b411 --- /dev/null +++ b/paddle/gserver/layers/MKLDNNAddtoLayer.h @@ -0,0 +1,130 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "MKLDNNLayer.h" +#include "mkldnn.hpp" + +namespace paddle { + +/** + * @brief A subclass of MKLDNNLayer Addto layer. + * + * The config file api is mkldnn_addto + */ +class MKLDNNAddtoLayer : public MKLDNNLayer { +protected: + std::vector inVals_; + std::vector inGrads_; + + // layer size == ic * ih * iw == oc * oh *ow, and can not be changed + size_t layerSize_; + + std::unique_ptr biases_; + + // buffers for adding bias + std::vector vals_; + std::vector grads_; + // primitives for adding bias + std::vector> fwdBias_; + std::shared_ptr bwdBias_; + +public: + explicit MKLDNNAddtoLayer(const LayerConfig& config) : MKLDNNLayer(config) {} + + ~MKLDNNAddtoLayer() {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void reshape( + int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) override; + + void resetFwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) override; + + void resetBwd(std::vector& pipeline, + MKLDNNMatrixPtr& in, + MKLDNNMatrixPtr& wgt, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out) override; + + void updateWeights(const UpdateCallback& callback) override; + + void printValueFormat() override { + for (size_t i = 0; i < inVals_.size(); ++i) { + VLOG(MKLDNN_FMTS) << i << " input: " << inVals_[i]->getFormat() << " >>>"; + } + if (outVal_) { + VLOG(MKLDNN_FMTS) << outVal_->getFormat() << " >>> "; + } + if (extOutVal_) { + VLOG(MKLDNN_FMTS) << extOutVal_->getFormat(); + } + } + + void printGradFormat() override { + if (extOutGrad_) { + VLOG(MKLDNN_FMTS) << extOutGrad_->getFormat(); + } + if (outGrad_) { + VLOG(MKLDNN_FMTS) << outGrad_->getFormat() << " <<< "; + } + for (size_t i = 0; i < inGrads_.size(); ++i) { + VLOG(MKLDNN_FMTS) << i << " input: " << inGrads_[i]->getFormat() << "<<<"; + } + } + +protected: + /** + * Forward functions: reset buffers(inputs, output, bias), + * reset primitive descriptor, + * reset pipeline. + */ + void resetFwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out); + void resetFwdPD(std::shared_ptr& pd, + std::shared_ptr& biasPD, + std::vector& inputs, + MKLDNNMatrixPtr bias, + MKLDNNMatrixPtr out); + void resetFwdPipeline(std::vector& pipeline, + std::shared_ptr& pd, + std::shared_ptr& biasPD, + std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out); + + /** + * Backward functions: reset buffers(inputs, output, bias) + */ + void resetBwdBuffers(std::vector& inputs, + MKLDNNMatrixPtr& bias, + MKLDNNMatrixPtr& out); + + /** + * prepare for bias + */ + void prepareBias(MKLDNNMatrixPtr& bias, + const MatrixPtr& biasMat, + const MKLDNNMatrixPtr& out, + std::vector& outs); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp b/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp index 9b0ae20f089e34a719883bc65e88e33ab9334e39..071bdf54d5dc9538d5ced580a73b9c0fbcea41fb 100644 --- a/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp +++ b/paddle/gserver/layers/MKLDNNBatchNormLayer.cpp @@ -119,13 +119,12 @@ void MKLDNNBatchNormLayer::reshape( int& bs, int& ic, int& ih, int& iw, int oc, int& oh, int& ow) { reshapeInput(bs, ih, iw); oh = ih; - ow = ow; + ow = iw; // ic_ and oc can not be changed CHECK_EQ(inputElemenCnt_ / bs / ih / iw, (size_t)ic) << "Input channel can not be changed"; reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - printSizeInfo(); } void MKLDNNBatchNormLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index b8120eda1e2dadab943869a05546351a369af6fd..8aa54e0a9efa7adb766cbb6009f6a29410c6ae7d 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -102,8 +102,6 @@ void MKLDNNConvLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - - printSizeInfo(); } void MKLDNNConvLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNConvLayer.h b/paddle/gserver/layers/MKLDNNConvLayer.h index 1fed0e1c6565b763a3ee73a0853f560ddfbd44c6..9c69136684e5f9005860b476ec6ed1bbc9ceff6c 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.h +++ b/paddle/gserver/layers/MKLDNNConvLayer.h @@ -92,7 +92,7 @@ public: void printSizeInfo() override { MKLDNNLayer::printSizeInfo(); VLOG(MKLDNN_SIZES) << getName() << ": fh: " << fh_ << ", fw: " << fw_ - << ": ph: " << ph_ << ", pw: " << pw_ << ", sh: " << sh_ + << ", ph: " << ph_ << ", pw: " << pw_ << ", sh: " << sh_ << ", sw: " << sw_ << ", dh: " << dh_ << ", dw: " << dw_; } diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index d82063a7130ca928ba042e210eb216f90c7207cd..350ec65fffbc73c3a6e4245f763f4c6aa868f574 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -60,18 +60,16 @@ void MKLDNNFcLayer::convertWeightsFromPaddle() { } CHECK(wgtVal_) << "should have been initialized"; - bool hasNoSpatial_ = ih_ == 1 && iw_ == 1; auto targetDim = wgtVal_->getDims(); - auto srcFmt = hasNoSpatial_ ? format::io : format::ihwo; + auto srcFmt = targetDim.size() == 2 ? format::io : format::ihwo; wgtVal_->reorderDataFrom(wgtVal_, srcFmt, targetDim); hasInitedWgt_ = true; } void MKLDNNFcLayer::convertWeightsToPaddle() { CHECK(wgtVal_) << "should have been initialized"; - bool hasNoSpatial_ = ih_ == 1 && iw_ == 1; auto targetDim = wgtVal_->getDims(); - auto dstFmt = hasNoSpatial_ ? format::io : format::ihwo; + auto dstFmt = targetDim.size() == 2 ? format::io : format::ihwo; wgtVal_->reorderDataTo(wgtVal_, dstFmt, targetDim); } @@ -86,8 +84,6 @@ void MKLDNNFcLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc); - - printSizeInfo(); } void MKLDNNFcLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/MKLDNNLayer.cpp b/paddle/gserver/layers/MKLDNNLayer.cpp index 663a10509857ec9fb487c1cda1621bdfac1250ac..e75ac5ba4647a8267b7bc189893bd7adb5c3053f 100644 --- a/paddle/gserver/layers/MKLDNNLayer.cpp +++ b/paddle/gserver/layers/MKLDNNLayer.cpp @@ -77,7 +77,7 @@ void MKLDNNLayer::forward(PassType passType) { needResetBwd_ = true; } - if (inputLayers_[0]->getType() == "data") { + if (inputLayers_[0]->getType() == "data" && inputLayers_.size() == 1) { // Update input value data when input layer is "data" type, // since the input value data address might be changed. CHECK(extInVal_); @@ -171,29 +171,27 @@ void MKLDNNLayer::resetWithMatrix(MKLDNNMatrixPtr& dnn, } void MKLDNNLayer::resetInValue( - MKLDNNMatrixPtr& in, const std::shared_ptr& intPD) { + MKLDNNMatrixPtr& in, + const std::shared_ptr& intPD, + size_t inputIdx) { cvtInVal_ = nullptr; extInVal_ = nullptr; in = nullptr; CHECK_GT(bs_ * ic_ * ih_ * iw_, 0); auto extPD = MKLDNNMatrix::createPrimitiveDesc( {bs_, ic_, ih_, iw_}, format::nchw, engine_); - const MatrixPtr& inMat = inputLayers_[0]->getOutputValue(); - in = std::dynamic_pointer_cast(inMat); - CHECK_EQ(inputIsOnlyMKLDNN(), in != nullptr); - if (in == nullptr || in->getFormat() == format::nc) { - in = MKLDNNMatrix::create(extPD, inMat); - } - extInVal_ = isPaddleFormat(in->getFormat()) ? in : nullptr; - if (in->getFormat() == format::nc) { - CHECK(ih_ == 1 && iw_ == 1); + const MatrixPtr& inMat = inputLayers_[inputIdx]->getOutputValue(); + extInVal_ = std::dynamic_pointer_cast(inMat); + CHECK_EQ(inputIsOnlyMKLDNN(), extInVal_ != nullptr); + if (extInVal_ == nullptr || extInVal_->getFormat() == format::nc) { + extInVal_ = MKLDNNMatrix::create(extPD, inMat); } + in = extInVal_; if (nullptr == intPD || in->getPrimitiveDesc() == *intPD) { return; } // need create reorder in = MKLDNNMatrix::create(*intPD); - extInVal_ = extInVal_ ? extInVal_ : MKLDNNMatrix::create(extPD, inMat); cvtInVal_ = MKLDNNMatrix::createReorder(extInVal_, in); CHECK(cvtInVal_) << "should not be emptry"; } @@ -216,11 +214,12 @@ void MKLDNNLayer::resetOutValue(MKLDNNMatrixPtr& out, } void MKLDNNLayer::resetInGrad(MKLDNNMatrixPtr& in, - memory::primitive_desc intPD) { + memory::primitive_desc intPD, + size_t inputIdx) { cvtInGrad_ = nullptr; extInGrad_ = nullptr; in = nullptr; - LayerPtr& input = inputLayers_[0]; + LayerPtr& input = inputLayers_[inputIdx]; if (input->getOutputGrad() == nullptr) { // no need input grad return; @@ -245,7 +244,6 @@ void MKLDNNLayer::resetInGrad(MKLDNNMatrixPtr& in, return; } // need create reorder - // TODO(TJ): add macro definition to simplify it CHECK(extInVal_ != nullptr && isPaddleFormat(extInVal_->getFormat())) << "should have external input value and the format must be nchw(nc)"; extInGrad_ = MKLDNNMatrix::create(extInVal_->getPrimitiveDesc(), inMat); @@ -289,7 +287,7 @@ void MKLDNNLayer::resetMergeGrad(MKLDNNMatrixPtr& out) { return; } CHECK(out) << "should have reset internal ouput grad"; - std::vector scales(outputMap_.size(), 1.0); + std::vector scales(outputMap_.size(), 1.0); std::vector srcPDs; std::vector srcs; for (auto it = outputMap_.begin(); it != outputMap_.end(); ++it) { diff --git a/paddle/gserver/layers/MKLDNNLayer.h b/paddle/gserver/layers/MKLDNNLayer.h index 2c21a5b2aaecb17a52a5de9a98664068f2255d83..7479c34c92b5231b2521493bc631474d4efd4224 100644 --- a/paddle/gserver/layers/MKLDNNLayer.h +++ b/paddle/gserver/layers/MKLDNNLayer.h @@ -199,7 +199,8 @@ protected: */ void resetInValue( MKLDNNMatrixPtr& in, - const std::shared_ptr& intPD = nullptr); + const std::shared_ptr& intPD = nullptr, + size_t inputIdx = 0); /** * reset output value from internal primitive desc. @@ -212,7 +213,9 @@ protected: * reset input grad from internal primitive desc. * reset both internal and external buffer and create reorder if necessary. */ - void resetInGrad(MKLDNNMatrixPtr& in, mkldnn::memory::primitive_desc intPD); + void resetInGrad(MKLDNNMatrixPtr& in, + mkldnn::memory::primitive_desc intPD, + size_t inputIdx = 0); /** * reset output grad from internal primitive desc. diff --git a/paddle/gserver/layers/MKLDNNPoolLayer.cpp b/paddle/gserver/layers/MKLDNNPoolLayer.cpp index 6e89260f49979d4edb4da138507a73dc2bf120de..a18c455beab96ef25b5545281bae4d48cec98d9e 100644 --- a/paddle/gserver/layers/MKLDNNPoolLayer.cpp +++ b/paddle/gserver/layers/MKLDNNPoolLayer.cpp @@ -71,8 +71,6 @@ void MKLDNNPoolLayer::reshape( reshapeOutput(oh, ow); resizeOutput(bs, oc * oh * ow); - - printSizeInfo(); } void MKLDNNPoolLayer::resetFwd(std::vector& pipeline, diff --git a/paddle/gserver/layers/ROIPoolLayer.cpp b/paddle/gserver/layers/ROIPoolLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..35d4b12d3d357800fe72899069b5377c252fac5f --- /dev/null +++ b/paddle/gserver/layers/ROIPoolLayer.cpp @@ -0,0 +1,220 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ROIPoolLayer.h" + +namespace paddle { + +REGISTER_LAYER(roi_pool, ROIPoolLayer); + +bool ROIPoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + + const ROIPoolConfig& layerConf = config_.inputs(0).roi_pool_conf(); + pooledWidth_ = layerConf.pooled_width(); + pooledHeight_ = layerConf.pooled_height(); + spatialScale_ = layerConf.spatial_scale(); + + return true; +} + +void ROIPoolLayer::forward(PassType passType) { + Layer::forward(passType); + + const ROIPoolConfig& layerConf = config_.inputs(0).roi_pool_conf(); + height_ = getInput(0).getFrameHeight(); + if (!height_) height_ = layerConf.height(); + width_ = getInput(0).getFrameWidth(); + if (!width_) width_ = layerConf.width(); + channels_ = getInputValue(0)->getWidth() / width_ / height_; + + size_t batchSize = getInput(0).getBatchSize(); + size_t numROIs = getInput(1).getBatchSize(); + + MatrixPtr dataValue = getInputValue(0); + MatrixPtr roiValue = getInputValue(1); + resetOutput(numROIs, channels_ * pooledHeight_ * pooledWidth_); + MatrixPtr outputValue = getOutputValue(); + + if (useGpu_) { // TODO(guosheng): implement on GPU later + MatrixPtr dataCpuBuffer; + Matrix::resizeOrCreate(dataCpuBuffer, + dataValue->getHeight(), + dataValue->getWidth(), + false, + false); + MatrixPtr roiCpuBuffer; + Matrix::resizeOrCreate(roiCpuBuffer, + roiValue->getHeight(), + roiValue->getWidth(), + false, + false); + dataCpuBuffer->copyFrom(*dataValue); + roiCpuBuffer->copyFrom(*roiValue); + dataValue = dataCpuBuffer; + roiValue = roiCpuBuffer; + MatrixPtr outputCpuBuffer; + Matrix::resizeOrCreate(outputCpuBuffer, + outputValue->getHeight(), + outputValue->getWidth(), + false, + false); + outputCpuBuffer->copyFrom(*outputValue); + outputValue = outputCpuBuffer; + } + + real* bottomData = dataValue->getData(); + size_t batchOffset = dataValue->getWidth(); + size_t channelOffset = height_ * width_; + real* bottomROIs = roiValue->getData(); + size_t roiOffset = roiValue->getWidth(); + size_t poolChannelOffset = pooledHeight_ * pooledWidth_; + + real* outputData = outputValue->getData(); + Matrix::resizeOrCreate(maxIdxs_, + numROIs, + channels_ * pooledHeight_ * pooledWidth_, + false, + false); + real* argmaxData = maxIdxs_->getData(); + + for (size_t n = 0; n < numROIs; ++n) { + // the first five elememts of each RoI should be: + // batch_idx, roi_x_start, roi_y_start, roi_x_end, roi_y_end + size_t roiBatchIdx = bottomROIs[0]; + size_t roiStartW = round(bottomROIs[1] * spatialScale_); + size_t roiStartH = round(bottomROIs[2] * spatialScale_); + size_t roiEndW = round(bottomROIs[3] * spatialScale_); + size_t roiEndH = round(bottomROIs[4] * spatialScale_); + CHECK_GE(roiBatchIdx, 0UL); + CHECK_LT(roiBatchIdx, batchSize); + size_t roiHeight = std::max(roiEndH - roiStartH + 1, 1UL); + size_t roiWidth = std::max(roiEndW - roiStartW + 1, 1UL); + real binSizeH = + static_cast(roiHeight) / static_cast(pooledHeight_); + real binSizeW = + static_cast(roiWidth) / static_cast(pooledWidth_); + real* batchData = bottomData + batchOffset * roiBatchIdx; + for (size_t c = 0; c < channels_; ++c) { + for (size_t ph = 0; ph < pooledHeight_; ++ph) { + for (size_t pw = 0; pw < pooledWidth_; ++pw) { + size_t hstart = static_cast(std::floor(ph * binSizeH)); + size_t wstart = static_cast(std::floor(pw * binSizeW)); + size_t hend = static_cast(std::ceil((ph + 1) * binSizeH)); + size_t wend = static_cast(std::ceil((pw + 1) * binSizeW)); + hstart = std::min(std::max(hstart + roiStartH, 0UL), height_); + wstart = std::min(std::max(wstart + roiStartW, 0UL), width_); + hend = std::min(std::max(hend + roiStartH, 0UL), height_); + wend = std::min(std::max(wend + roiStartW, 0UL), width_); + + bool isEmpty = (hend <= hstart) || (wend <= wstart); + size_t poolIndex = ph * pooledWidth_ + pw; + if (isEmpty) { + outputData[poolIndex] = 0; + argmaxData[poolIndex] = -1; + } + + for (size_t h = hstart; h < hend; ++h) { + for (size_t w = wstart; w < wend; ++w) { + size_t index = h * width_ + w; + if (batchData[index] > outputData[poolIndex]) { + outputData[poolIndex] = batchData[index]; + argmaxData[poolIndex] = index; + } + } + } + } + } + batchData += channelOffset; + outputData += poolChannelOffset; + argmaxData += poolChannelOffset; + } + bottomROIs += roiOffset; + } + if (useGpu_) { + getOutputValue()->copyFrom(*outputValue); + } +} + +void ROIPoolLayer::backward(const UpdateCallback& callback) { + MatrixPtr inGradValue = getInputGrad(0); + MatrixPtr outGradValue = getOutputGrad(); + MatrixPtr roiValue = getInputValue(1); + + if (useGpu_) { + MatrixPtr inGradCpuBuffer; + Matrix::resizeOrCreate(inGradCpuBuffer, + inGradValue->getHeight(), + inGradValue->getWidth(), + false, + false); + MatrixPtr outGradCpuBuffer; + Matrix::resizeOrCreate(outGradCpuBuffer, + outGradValue->getHeight(), + outGradValue->getWidth(), + false, + false); + MatrixPtr roiCpuBuffer; + Matrix::resizeOrCreate(roiCpuBuffer, + roiValue->getHeight(), + roiValue->getWidth(), + false, + false); + inGradCpuBuffer->copyFrom(*inGradValue); + outGradCpuBuffer->copyFrom(*outGradValue); + roiCpuBuffer->copyFrom(*roiValue); + inGradValue = inGradCpuBuffer; + outGradValue = outGradCpuBuffer; + roiValue = roiCpuBuffer; + } + + real* bottomROIs = roiValue->getData(); + size_t numROIs = getInput(1).getBatchSize(); + size_t roiOffset = getInputValue(1)->getWidth(); + + real* inDiffData = inGradValue->getData(); + size_t batchOffset = getInputValue(0)->getWidth(); + size_t channelOffset = height_ * width_; + + real* outDiffData = outGradValue->getData(); + size_t poolChannelOffset = pooledHeight_ * pooledWidth_; + real* argmaxData = maxIdxs_->getData(); + + for (size_t n = 0; n < numROIs; ++n) { + size_t roiBatchIdx = bottomROIs[0]; + real* batchDiffData = inDiffData + batchOffset * roiBatchIdx; + for (size_t c = 0; c < channels_; ++c) { + for (size_t ph = 0; ph < pooledHeight_; ++ph) { + for (size_t pw = 0; pw < pooledWidth_; ++pw) { + size_t poolIndex = ph * pooledWidth_ + pw; + if (argmaxData[poolIndex] > 0) { + size_t index = static_cast(argmaxData[poolIndex]); + batchDiffData[index] += outDiffData[poolIndex]; + } + } + } + batchDiffData += channelOffset; + outDiffData += poolChannelOffset; + argmaxData += poolChannelOffset; + } + bottomROIs += roiOffset; + } + + if (useGpu_) { + getInputGrad(0)->copyFrom(*inGradValue); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ROIPoolLayer.h b/paddle/gserver/layers/ROIPoolLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..4f07e49d6fd1eda9fa7bd46e4cec771a75f571be --- /dev/null +++ b/paddle/gserver/layers/ROIPoolLayer.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" + +namespace paddle { + +/** + * A layer used by Fast R-CNN to extract feature maps of ROIs from the last + * feature map. + * - Input: This layer needs two input layers: The first input layer is a + * convolution layer; The second input layer contains the ROI data + * which is the output of ProposalLayer in Faster R-CNN. layers for + * generating bbox location offset and the classification confidence. + * - Output: The ROIs' feature map. + * Reference: + * Shaoqing Ren, Kaiming He, Ross Girshick, and Jian Sun. + * Faster R-CNN: Towards Real-Time Object Detection with Region Proposal + * Networks + */ + +class ROIPoolLayer : public Layer { +protected: + size_t channels_; + size_t width_; + size_t height_; + size_t pooledWidth_; + size_t pooledHeight_; + real spatialScale_; + + // Since there is no int matrix, use real maxtrix instead. + MatrixPtr maxIdxs_; + +public: + explicit ROIPoolLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void forward(PassType passType) override; + void backward(const UpdateCallback& callback = nullptr) override; +}; +} // namespace paddle diff --git a/paddle/gserver/layers/ScaleSubRegionLayer.cpp b/paddle/gserver/layers/ScaleSubRegionLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aa6778aef4e893208fd064ca22e217c6c4d960f9 --- /dev/null +++ b/paddle/gserver/layers/ScaleSubRegionLayer.cpp @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "ScaleSubRegionLayer.h" +#include "paddle/utils/Stat.h" +namespace paddle { + +REGISTER_LAYER(scale_sub_region, ScaleSubRegionLayer); + +bool ScaleSubRegionLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(static_cast(inputLayers_.size()), 2); + auto& conf = config_.inputs(0).scale_sub_region_conf(); + value_ = conf.value(); + + createFunction(forward_, "ScaleSubRegion", FuncConfig().set("value", value_)); + createFunction( + backward_, "ScaleSubRegionGrad", FuncConfig().set("value", value_)); + + return true; +} + +void ScaleSubRegionLayer::forward(PassType passType) { + Layer::forward(passType); + auto in0 = getInput(0); + imgH_ = in0.getFrameHeight(); + imgW_ = in0.getFrameWidth(); + if (imgH_ == 0 || imgW_ == 0) { + auto& conf = config_.inputs(0).scale_sub_region_conf(); + imgH_ = conf.image_conf().img_size_y(); + imgW_ = conf.image_conf().img_size(); + } + MatrixPtr imgV = in0.value; + size_t batchSize = imgV->getHeight(); + size_t spatialSize = imgH_ * imgW_; + channelsNum_ = imgV->getWidth() / spatialSize; + shape_ = TensorShape({batchSize, channelsNum_, imgH_, imgW_}); + + resetOutput(batchSize, imgV->getWidth()); + auto& out = getOutput(); + out.setFrameHeight(imgH_); + out.setFrameWidth(imgW_); + + MatrixPtr indicesV = getInputValue(1); + indicesShape_ = TensorShape({batchSize, 6}); + + REGISTER_TIMER_INFO("ScaleSubRegionForward", getName().c_str()); + BufferArgs inArgs; + BufferArgs outArgs; + inArgs.addArg(*imgV, shape_); + inArgs.addArg(*indicesV, indicesShape_); + outArgs.addArg(*out.value, shape_, ASSIGN_TO); + forward_[0]->calc(inArgs, outArgs); +} + +void ScaleSubRegionLayer::backward(const UpdateCallback& callback) { + REGISTER_TIMER_INFO("ScaleSubRegionBackward", getName().c_str()); + BufferArgs inArgs; + BufferArgs outArgs; + inArgs.addArg(*getOutputGrad(), shape_); + inArgs.addArg(*getInputValue(1), indicesShape_); + outArgs.addArg(*getInputGrad(0), shape_, ADD_TO); + backward_[0]->calc(inArgs, outArgs); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ScaleSubRegionLayer.h b/paddle/gserver/layers/ScaleSubRegionLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..a27c56de93bb6fdde0f95cd4c5abe5dfabe4e858 --- /dev/null +++ b/paddle/gserver/layers/ScaleSubRegionLayer.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" + +namespace paddle { + +/** + * \brief For each instance, this layer can be used to multiply a value to a + * specified sub continuous region. By providing start index and end + * index for C/H/W, you can specify the location and shape of the + * region. + * + * input_0: Input value. + * input_1: Indices value to specify the location an shape of the + * region. + */ +class ScaleSubRegionLayer : public Layer { +public: + explicit ScaleSubRegionLayer(const LayerConfig& config) : Layer(config) {} + + ~ScaleSubRegionLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + + void backward(const UpdateCallback& callback = nullptr); + +protected: + TensorShape shape_; + TensorShape indicesShape_; + size_t imgH_; + size_t imgW_; + size_t channelsNum_; + real value_; +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/SubSequenceLayer.cpp b/paddle/gserver/layers/SubSequenceLayer.cpp index 19b7ad1869af98e6313fe85a40203fd1e84f31d6..00d8ce017aa0121217688a1afc1fe31b4c3619ec 100644 --- a/paddle/gserver/layers/SubSequenceLayer.cpp +++ b/paddle/gserver/layers/SubSequenceLayer.cpp @@ -98,8 +98,19 @@ void SubSequenceLayer::forward(PassType passType) { CHECK_EQ(numSequences2, numSequences3); MatrixPtr inputValue = input.value; - IVectorPtr offsetValue = offsetSeq.ids; - IVectorPtr sizeValue = sizeSeq.ids; + IVectorPtr offsetValue; + IVectorPtr sizeValue; + + if (useGpu_) { + // copy to cpu + IVector::resizeOrCreate(offsetValue, offsetSeq.ids->getSize(), false); + IVector::resizeOrCreate(sizeValue, sizeSeq.ids->getSize(), false); + offsetValue->copyFrom(*offsetSeq.ids); + sizeValue->copyFrom(*sizeSeq.ids); + } else { + offsetValue = offsetSeq.ids; + sizeValue = sizeSeq.ids; + } CHECK_EQ(offsetValue->getSize(), numSequences1); CHECK_EQ(sizeValue->getSize(), numSequences1); @@ -176,8 +187,21 @@ void SubSequenceLayer::backward(const UpdateCallback& callback) { size_t numSequences1 = startPositions1->getSize() - 1; const int* starts1 = startPositions1->getData(); - IVectorPtr offsetValue = getInput(1).ids; - IVectorPtr sizeValue = getInput(2).ids; + const Argument& offsetSeq = getInput(1); + const Argument& sizeSeq = getInput(2); + IVectorPtr offsetValue; + IVectorPtr sizeValue; + + if (useGpu_) { + // copy to cpu + IVector::resizeOrCreate(offsetValue, offsetSeq.ids->getSize(), false); + IVector::resizeOrCreate(sizeValue, sizeSeq.ids->getSize(), false); + offsetValue->copyFrom(*offsetSeq.ids); + sizeValue->copyFrom(*sizeSeq.ids); + } else { + offsetValue = offsetSeq.ids; + sizeValue = sizeSeq.ids; + } int* offsets = offsetValue->getData(); int* sizes = sizeValue->getData(); diff --git a/paddle/gserver/tests/MKLDNNTester.cpp b/paddle/gserver/tests/MKLDNNTester.cpp index 73b7e8857f35d194e71b2b5b341f89b77fd1f8b0..afe1608eab8eaf1217a7a0c8a2774e37c5ea83f4 100644 --- a/paddle/gserver/tests/MKLDNNTester.cpp +++ b/paddle/gserver/tests/MKLDNNTester.cpp @@ -132,7 +132,7 @@ void MKLDNNTester::checkForward() { VLOG(MKLDNN_TESTS) << "Check Forward"; printTopDatas(); double delta = - compareMatrix(dnnLayer_->getOutputValue(), refLayer_->getOutputValue()); + compareMatrix(refLayer_->getOutputValue(), dnnLayer_->getOutputValue()); EXPECT_LE(fabs(delta), eps_); } @@ -147,7 +147,7 @@ void MKLDNNTester::checkBackwardData() { VLOG(MKLDNN_ALL) << "Reference Backward Result: InputGrad " << i; printMatrix(refDiff); - double delta = compareMatrix(dnnDiff, refDiff); + double delta = compareMatrix(refDiff, dnnDiff); EXPECT_LE(fabs(delta), eps_); if (isBN) { // the other two inputs in batch norm are for moving mean and var @@ -177,7 +177,7 @@ void MKLDNNTester::checkBackwardWgts() { << parameters_[REF][i]->getName(); printVector(ref); - double delta = compareVector(dnn, ref); + double delta = compareVector(ref, dnn); EXPECT_LE(fabs(delta), eps_); } @@ -273,31 +273,37 @@ void MKLDNNTester::printVector(const VectorPtr& v) { VLOG(MKLDNN_ALL) << std::endl << ostr.str(); } -double MKLDNNTester::getDelta(const real* d1, - const real* d2, +double MKLDNNTester::getDelta(const real* refer, + const real* value, size_t len, const float failRate, const float thres) { double delta = 0, sum = 0; int failCnt = 0; const double eps = 1e-5; - double maxOut = 0; + double maxRatio = 0; for (size_t i = 0; i < len; ++i) { - double ref = fabs(d2[i]); - double diff = fabs(d1[i] - d2[i]); + double ref = fabs(refer[i]); + double val = fabs(value[i]); + double diff = fabs(refer[i] - value[i]); delta += diff; sum += ref; - if (ref > eps && fabs(d1[i]) > eps && diff / ref > thres) { - maxOut = std::max(maxOut, diff / ref); + if (ref < eps && val < eps) { // both values are very small + continue; + } + double ratio = diff / ref; + if (ratio > thres) { + maxRatio = std::max(maxRatio, ratio); failCnt++; } } - EXPECT_TRUE(std::isnormal(sum)); EXPECT_FALSE(std::isinf(sum)); + EXPECT_FALSE(std::isnan(sum)); EXPECT_FALSE(std::isnan(delta)); VLOG(MKLDNN_ALL) << "reference avg data: " << sum / len << ", delta: " << delta / sum << ", failCnt:" << failCnt; - return (failCnt / (float)len) > failRate ? maxOut : delta / sum; + double res = sum > eps ? delta / sum : eps; + return (failCnt / (float)len) > failRate ? maxRatio : res; } double MKLDNNTester::compareMatrix(const MatrixPtr& m1, const MatrixPtr& m2) { @@ -515,12 +521,16 @@ void MKLDNNTester::getOutResult(const std::string& configPath, gradientMachine->forward(in.inArgs[i], &outArgs, PASS_TRAIN); // save forward result for (size_t k = 0; k < outArgs.size(); k++) { - MatrixPtr value = Matrix::create(outArgs[k].value->getHeight(), - outArgs[k].value->getWidth(), - false, - false); - value->copyFrom(*outArgs[k].value); - out.outValues.push_back(value); + const MatrixPtr& src = outArgs[k].value; + MatrixPtr dst = + Matrix::create(src->getHeight(), src->getWidth(), false, false); + if (typeid(*src) == typeid(MKLDNNMatrix)) { + MKLDNNMatrixPtr dnnSrc = std::dynamic_pointer_cast(src); + dnnSrc->copyTo(*dst); + } else { + dst->copyFrom(*src); + } + out.outValues.push_back(dst); } // random backward input @@ -543,19 +553,19 @@ void MKLDNNTester::getOutResult(const std::string& configPath, void MKLDNNTester::compareResult(DataOut& ref, DataOut& dnn, float eps) { CHECK_EQ(ref.outValues.size(), dnn.outValues.size()); CHECK_EQ(ref.paraValues.size(), dnn.paraValues.size()); - VLOG(MKLDNN_TESTS) << "compare value size: " << ref.outValues.size(); for (size_t i = 0; i < ref.outValues.size(); i++) { + VLOG(MKLDNN_TESTS) << "compare value index: " << i; EXPECT_LE(fabs(compareMatrix(ref.outValues[i], dnn.outValues[i])), eps); } - VLOG(MKLDNN_TESTS) << "compare param size: " << ref.outValues.size(); for (size_t i = 0; i < ref.paraValues.size(); i++) { + VLOG(MKLDNN_TESTS) << "compare param index: " << i; EXPECT_LE(fabs(compareVector(ref.paraValues[i], dnn.paraValues[i])), eps); } } -void MKLDNNTester::runBranchesTest(const std::string& configPath, - size_t iter, - float eps) { +void MKLDNNTester::runNetTest(const std::string& configPath, + size_t iter, + float eps) { DataIn in; initArgument(in, configPath, iter); DataOut outCpu, outDnn; diff --git a/paddle/gserver/tests/MKLDNNTester.h b/paddle/gserver/tests/MKLDNNTester.h index 19d8848f74f2ee4a809e42164a0eb180abd2a4e1..ca55a45bc77b4e171619ab788d7c7dfeefcd036a 100644 --- a/paddle/gserver/tests/MKLDNNTester.h +++ b/paddle/gserver/tests/MKLDNNTester.h @@ -85,17 +85,17 @@ public: bool printDetails = false, size_t iter = 3, float epsilon = 1e-4); - static void runBranchesTest(const std::string& configPath, - size_t iter = 3, - float eps = 1e-4); + static void runNetTest(const std::string& configPath, + size_t iter = 2, + float eps = 1e-4); static void initArgument(DataIn& data, const std::string& configPath, - size_t iter = 3); + size_t iter = 2); static void getOutResult(const std::string& configPath, DataIn& in, DataOut& out, bool use_mkldnn, - size_t iter = 3); + size_t iter = 2); private: void reset(const TestConfig& dnn, const TestConfig& ref, size_t batchSize); @@ -128,13 +128,13 @@ private: /** * Get delta percent - * if many(>failRate) wrong(abs(dnn-ref)/abs(ref)>thres) points return the - * max(diff/ref) - * else return sum(abs(a-b)) / sum(abs(b)) + * if many(>failRate) wrong(abs(val-ref)/abs(ref) > thres) points + * return the max(diff/ref) + * else return sum(abs(diff)) / sum(abs(ref)) * The return value should be smaller than eps when passing. */ - static double getDelta(const real* d1, - const real* d2, + static double getDelta(const real* refer, + const real* value, size_t len, const float failRate = 1e-3, const float thres = 0.1); diff --git a/paddle/gserver/tests/mkldnn_branch_net.conf b/paddle/gserver/tests/mkldnn_branch_net.conf new file mode 100644 index 0000000000000000000000000000000000000000..8d5146abb0ebd7f5d6c512457f3cb5c84eac20f5 --- /dev/null +++ b/paddle/gserver/tests/mkldnn_branch_net.conf @@ -0,0 +1,142 @@ +# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=16) +channels = get_config_arg("channels", int, 2) + +def two_conv(input, group_name): + out1 = img_conv_layer(input=input, + name=group_name+'_conv1_', + filter_size=1, + num_filters=channels, + padding=0, + shared_biases=True, + act=ReluActivation()) + + out2 = img_conv_layer(input=input, + name=group_name+'_conv2_', + filter_size=3, + num_filters=channels, + padding=1, + shared_biases=True, + act=ReluActivation()) + return out1, out2 + +def two_conv_bn(input, group_name): + out1, out2 = two_conv(input, group_name) + out1 = batch_norm_layer(input=out1, + name=group_name+'_bn1_', + use_global_stats=False, + act=ReluActivation()) + + out2 = batch_norm_layer(input=out2, + name=group_name+'_bn2_', + use_global_stats=False, + act=ReluActivation()) + return out1, out2 + +def two_conv_pool(input, group_name): + out1, out2 = two_conv(input, group_name) + out1 = img_pool_layer(input=out1, + name=group_name+'_pool1_', + pool_size=3, + stride=2, + padding=0, + pool_type=MaxPooling()) + + out2 = img_pool_layer(input=out2, + name=group_name+'_pool2_', + pool_size=5, + stride=2, + padding=1, + pool_type=MaxPooling()) + return out1, out2 + +def two_fc(input, group_name): + out1 = fc_layer(input=input, + name=group_name+'_fc1_', + size=channels, + bias_attr=False, + act=LinearActivation()) + + out2 = fc_layer(input=input, + name=group_name+'_fc2_', + size=channels, + bias_attr=False, + act=LinearActivation()) + return out1, out2 + +data = data_layer(name ="input", size=channels*16*16) + +tmp = img_conv_layer(input=data, + num_channels=channels, + filter_size=3, + num_filters=channels, + padding=1, + shared_biases=True, + act=ReluActivation()) + +a1, a2 = two_conv(tmp, 'conv_branch') +tmp = addto_layer(input=[a1, a2], + act=ReluActivation(), + bias_attr=False) + +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=2, + padding=1, + pool_type=AvgPooling()) + +b1, b2 = two_conv_pool(tmp, 'pool_branch') +tmp = concat_layer(input=[b1, b2]) + +tmp = img_pool_layer(input=tmp, + num_channels=channels*2, + pool_size=3, + stride=2, + padding=1, + pool_type=MaxPooling()) + +tmp = img_conv_layer(input=tmp, + filter_size=3, + num_filters=channels, + padding=1, + stride=2, + shared_biases=True, + act=LinearActivation(), + bias_attr=False) + +tmp = batch_norm_layer(input=tmp, + use_global_stats=False, + act=ReluActivation()) + +c1, c2 = two_conv_bn(tmp, 'bn_branch') +tmp = addto_layer(input=[c1, c2], + act=ReluActivation(), + bias_attr=False) + +tmp = fc_layer(input=tmp, size=channels, + bias_attr=True, + act=ReluActivation()) + +d1, d2 = two_fc(tmp, 'fc_branch') +tmp = addto_layer(input=[d1, d2]) + +out = fc_layer(input=tmp, size=10, + bias_attr=True, + act=SoftmaxActivation()) + +outputs(out) diff --git a/paddle/gserver/tests/mkldnn_branches_fc.conf b/paddle/gserver/tests/mkldnn_branches_fc.conf deleted file mode 100644 index fb85425c2b63c7604d636e2b0c5d20d91fb5de1b..0000000000000000000000000000000000000000 --- a/paddle/gserver/tests/mkldnn_branches_fc.conf +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -settings(batch_size=16) -channels = get_config_arg("channels", int, 2) - -def two_fc(input, group_name): - out1 = fc_layer(input=input, - name=group_name+'_fc1', - size=channels, - bias_attr=False, - act=LinearActivation()) - - out2 = fc_layer(input=input, - name=group_name+'_fc2', - size=channels, - bias_attr=False, - act=LinearActivation()) - return out1, out2 - -data = data_layer(name ="input", size=channels*16*16) - -conv = img_conv_layer(input=data, - num_channels=channels, - filter_size=3, - num_filters=channels, - padding=1, - shared_biases=True, - act=LinearActivation()) - -pool = img_pool_layer(input=conv, - pool_size=3, - stride=2, - padding=1, - pool_type=AvgPooling()) - -a1, a2 = two_fc(input=pool, group_name='a') - -concat = concat_layer(input=[a1, a2]) - -b1, b2 = two_fc(input=pool, group_name='b') - -addto = addto_layer(input=[b1, b2]) - -outputs([concat, addto]) diff --git a/paddle/gserver/tests/mkldnn_branches_pool.conf b/paddle/gserver/tests/mkldnn_branches_pool.conf deleted file mode 100644 index ca17c74752ab0777a69f818d9f43275a6140cb4c..0000000000000000000000000000000000000000 --- a/paddle/gserver/tests/mkldnn_branches_pool.conf +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from paddle.trainer_config_helpers import * - -settings(batch_size=16) -channels = get_config_arg("channels", int, 2) - -def two_pool(input, group_name): - out1 = img_pool_layer(input=input, - name=group_name+'_pool1', - pool_size=3, - stride=2, - padding=0, - pool_type=MaxPooling()) - - out2 = img_pool_layer(input=input, - name=group_name+'_pool2', - pool_size=5, - stride=2, - padding=1, - pool_type=MaxPooling()) - return out1, out2 - -data = data_layer(name ="input", size=channels*16*16) - -conv = img_conv_layer(input=data, - num_channels=channels, - filter_size=3, - num_filters=channels, - padding=1, - shared_biases=True, - act=LinearActivation()) - -pool = img_pool_layer(input=conv, - pool_size=3, - stride=1, - padding=1, - pool_type=AvgPooling()) - -a1, a2 = two_pool(input=pool, group_name='a') - -concat = concat_layer(input=[a1, a2]) - -b1, b2 = two_pool(input=pool, group_name='b') - -addto = addto_layer(input=[b1, b2]) - -outputs([concat, addto]) diff --git a/paddle/gserver/tests/mkldnn_branches_conv.conf b/paddle/gserver/tests/mkldnn_simple_net.conf similarity index 64% rename from paddle/gserver/tests/mkldnn_branches_conv.conf rename to paddle/gserver/tests/mkldnn_simple_net.conf index 2628509db43e6a5f69a4f5ea956bffdc2837e32a..8bbe91e56d0ba6da06475ad16f3162ee1103ee02 100644 --- a/paddle/gserver/tests/mkldnn_branches_conv.conf +++ b/paddle/gserver/tests/mkldnn_simple_net.conf @@ -17,40 +17,48 @@ from paddle.trainer_config_helpers import * settings(batch_size=16) channels = get_config_arg("channels", int, 2) -def two_conv(input, group_name): - out1 = img_conv_layer(input=input, - name=group_name+'_conv1', - filter_size=1, - num_filters=channels, - padding=0, - shared_biases=True, - act=ReluActivation()) +data = data_layer(name ="input", size=channels*16*16) - out2 = img_conv_layer(input=input, - name=group_name+'_conv2', +tmp = img_conv_layer(input=data, + num_channels=channels, filter_size=3, num_filters=channels, padding=1, shared_biases=True, act=ReluActivation()) - return out1, out2 -data = data_layer(name ="input", size=channels*16*16) +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=1, + padding=0, + pool_type=AvgPooling()) -conv = img_conv_layer(input=data, - num_channels=channels, +tmp = img_conv_layer(input=tmp, filter_size=3, num_filters=channels, padding=1, shared_biases=True, - act=ReluActivation()) + act=LinearActivation(), + bias_attr=False) -a1, a2 = two_conv(input=conv, group_name='a') +tmp = batch_norm_layer(input=tmp, + use_global_stats=False, + act=ReluActivation()) -concat = concat_layer(input=[a1, a2]) +tmp = img_pool_layer(input=tmp, + pool_size=3, + stride=2, + padding=1, + pool_type=MaxPooling()) -b1, b2 = two_conv(input=conv, group_name='b') +tmp = fc_layer(input=tmp, + size=channels, + bias_attr=False, + act=ReluActivation()) -addto = addto_layer(input=[b1, b2]) +out = fc_layer(input=tmp, + size=10, + bias_attr=True, + act=SoftmaxActivation()) -outputs([concat, addto]) +outputs(out) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 61f39926cf5fd330179a741c14f35b14cb1e2685..141298d26d70a251121e46e39c3204184b191199 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -53,7 +53,7 @@ TEST(Operator, dot_mul) { TEST(Projection, context) { for (auto contextStart : {-5, -3, -1, 0, 3}) { for (auto contextLength : {1, 2, 5, 7}) { - for (auto batchSize : {1, 2, 5, 20, 50}) { + for (auto batchSize : {1, 2, 5, 20}) { for (auto trainablePadding : {false, true}) { LOG(INFO) << " contextStart=" << contextStart << " contextLength=" << contextLength @@ -585,14 +585,14 @@ TEST(Layer, maxoutLayer) { } void testFcLayer(string format, size_t nnz) { TestConfig config; - config.biasSize = 4096; + config.biasSize = 1024; config.layerConfig.set_type("fc"); - config.layerConfig.set_size(4096); + config.layerConfig.set_size(1024); config.layerConfig.set_active_type("sigmoid"); config.layerConfig.set_drop_rate(0.1); config.inputDefs.push_back( - {INPUT_DATA, "layer_0", 8192, nnz, ParaSparse(format)}); + {INPUT_DATA, "layer_0", 2048, nnz, ParaSparse(format)}); config.layerConfig.add_inputs(); LOG(INFO) << config.inputDefs[0].sparse.sparse << " " @@ -609,9 +609,9 @@ void testFcLayer(string format, size_t nnz) { } TEST(Layer, fcLayer) { - testFcLayer("", 4096 * 4096 * 2); - testFcLayer("csc", 4096 * 40); - testFcLayer("csr", 4096 * 40); + testFcLayer("", 1024 * 1024 * 2); + testFcLayer("csc", 1024 * 10); + testFcLayer("csr", 1024 * 10); } TEST(Layer, SelectiveFullyConnectedLayer) { @@ -1995,7 +1995,7 @@ TEST(Layer, multibox_loss) { TEST(Layer, TransLayer) { TestConfig config; const int height = 128; - const int width = 1028; + const int width = 256; config.layerConfig.set_type("trans"); config.layerConfig.set_size(width); @@ -2056,6 +2056,43 @@ TEST(Layer, CropLayer) { } } +TEST(Layer, roi_pool) { + TestConfig config; + config.layerConfig.set_type("roi_pool"); + config.biasSize = 0; + LayerInputConfig* input = config.layerConfig.add_inputs(); + ROIPoolConfig* roiPoolConf = input->mutable_roi_pool_conf(); + roiPoolConf->set_pooled_width(7); + roiPoolConf->set_pooled_height(7); + roiPoolConf->set_spatial_scale(1. / 16); + roiPoolConf->set_width(14); + roiPoolConf->set_height(14); + + const size_t roiNum = 10; + const size_t roiDim = 10; + const size_t batchSize = 5; + MatrixPtr roiValue = Matrix::create(roiNum, roiDim, false, false); + roiValue->zeroMem(); + real* roiData = roiValue->getData(); + for (size_t i = 0; i < roiNum; ++i) { + roiData[i * roiDim + 0] = std::rand() % batchSize; + roiData[i * roiDim + 1] = std::rand() % 224; // xMin + roiData[i * roiDim + 2] = std::rand() % 224; // yMin + size_t xMin = static_cast(roiData[i * roiDim + 1]); + size_t yMin = static_cast(roiData[i * roiDim + 2]); + roiData[i * roiDim + 3] = xMin + std::rand() % (224 - xMin); // xMax + roiData[i * roiDim + 4] = yMin + std::rand() % (224 - yMin); // yMax + } + + config.inputDefs.push_back({INPUT_DATA, "input", 3 * 14 * 14, {}}); + config.inputDefs.push_back({INPUT_SELF_DEFINE_DATA, "rois", roiValue, {}}); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "roi_pool", batchSize, false, useGpu, false); + } +} + TEST(Layer, SwitchOrderLayer) { TestConfig config; // config input_0 @@ -2358,6 +2395,38 @@ TEST(Layer, ScaleShiftLayer) { } } +TEST(Layer, ScaleSubRegionLayer) { + const size_t batchSize = 64; + const size_t size = 4096; + TestConfig config; + config.layerConfig.set_type("scale_sub_region"); + config.inputDefs.push_back({INPUT_DATA, "input", size, 0}); + MatrixPtr indicesV = Matrix::create(batchSize, 6, false, false); + auto* data = indicesV->getData(); + for (size_t i = 0; i < batchSize; ++i) { + data[i * 2] = 2; + data[i * 2 + 1] = 4; + data[i * 2 + 2] = 16; + data[i * 2 + 3] = 32; + data[i * 2 + 4] = 16; + data[i * 2 + 5] = 32; + } + config.inputDefs.push_back({INPUT_SELF_DEFINE_DATA, "indices", indicesV, {}}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + ScaleSubRegionConfig* scaleSubRegionConf = + input->mutable_scale_sub_region_conf(); + ImageConfig* imgConf = scaleSubRegionConf->mutable_image_conf(); + imgConf->set_img_size(32); + imgConf->set_img_size_y(32); + imgConf->set_channels(4); + scaleSubRegionConf->set_value(2.0); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "scale_sub_region", batchSize, false, useGpu, false); + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 85d4f437c2664135a7975c6ed3270d8f1ddbeaf4..a859e34c8996d81f14bf1edcb6e23d5a4f687e6b 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -234,8 +234,7 @@ static void getMKLDNNBatchNormConfig(TestConfig& cfg, cfg.inputDefs.push_back({INPUT_DATA, "layer_2_moving_var", 1, size_t(pm.ic)}); cfg.inputDefs.back().isStatic = true; LayerInputConfig* input = cfg.layerConfig.add_inputs(); - // TODO(TJ): uncomment me when refine and support comparing all zeroes vector - // cfg.layerConfig.set_active_type("relu"); + cfg.layerConfig.set_active_type("relu"); cfg.layerConfig.add_inputs(); cfg.layerConfig.add_inputs(); ImageConfig* img_conf = input->mutable_image_conf(); @@ -270,22 +269,51 @@ void testBatchNormLayer(const testBatchNormDesc& pm) { TEST(MKLDNNLayer, BatchNormLayer) { testBatchNormLayer({4, 10, 6, 6}); testBatchNormLayer({16, 32, 16, 16}); + testBatchNormLayer({4, 16, 8, 10}); } -struct testActDesc { +struct testImageDesc { int bs, ic, ih, iw; }; -static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { +static void getAddtoConfig(TestConfig& cfg, + const testImageDesc& pm, + const size_t nInputs = 1) { cfg.biasSize = 0; cfg.layerConfig.set_type("addto"); size_t layerSize = pm.ic * pm.ih * pm.iw; cfg.layerConfig.set_size(layerSize); - cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); - cfg.layerConfig.add_inputs(); + cfg.layerConfig.set_active_type("relu"); + for (size_t i = 0; i < nInputs; ++i) { + std::stringstream ss; + ss << "layer_" << i; + cfg.inputDefs.push_back({INPUT_DATA, ss.str(), layerSize, 0}); + LayerInputConfig* input = cfg.layerConfig.add_inputs(); + ImageConfig* img_conf = input->mutable_image_conf(); + img_conf->set_channels(pm.ic); + img_conf->set_img_size_y(pm.ih); + img_conf->set_img_size(pm.iw); + } +} + +void testAddtoLayer(const testImageDesc& pm, const size_t nInputs) { + CHECK_GE(nInputs, 1UL); + TestConfig dnnConfig; + getAddtoConfig(dnnConfig, pm, nInputs); + dnnConfig.layerConfig.set_type("mkldnn_addto"); + for (auto withBias : {false, true}) { + dnnConfig.biasSize = withBias ? pm.ic * pm.ih * pm.iw : 0; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "addto", pm) + } +} + +TEST(MKLDNNLayer, AddtoLayer) { + testAddtoLayer({16, 5, 14, 14}, 1); + testAddtoLayer({8, 10, 8, 8}, 2); + testAddtoLayer({4, 12, 1, 1}, 3); } -void testActivation(std::string actType, const testActDesc& pm) { +void testActivation(std::string actType, const testImageDesc& pm) { // TODO(TJ): remove me when paddle support elu activation if (actType == "mkldnn_elu") { return; @@ -309,15 +337,15 @@ TEST(MKLDNNActivation, Activations) { } DECLARE_string(config_args); -TEST(MKLDNNLayer, branches) { - std::vector cases = {"conv", "pool", "fc"}; +TEST(MKLDNNNet, net) { + std::vector cases = {"simple", "branch"}; for (auto name : cases) { - std::string config = "./gserver/tests/mkldnn_branches_" + name + ".conf"; + std::string config = "./gserver/tests/mkldnn_" + name + "_net.conf"; for (auto channels : {2, 32}) { std::ostringstream oss; oss << "channels=" << channels; FLAGS_config_args = oss.str(); - MKLDNNTester::runBranchesTest(config); + MKLDNNTester::runNetTest(config); } } } diff --git a/paddle/math/MKLDNNMatrix.cpp b/paddle/math/MKLDNNMatrix.cpp index 21a8f73c3e650d4b3c3b86247594cd965f4ead35..a710479bab82ed52122cf59bb14a05ccbd4aa05c 100644 --- a/paddle/math/MKLDNNMatrix.cpp +++ b/paddle/math/MKLDNNMatrix.cpp @@ -152,12 +152,7 @@ void MKLDNNMatrix::downSpatial() { } memory::desc md = memory::desc(dstDims, getDtype(), dstFmt); memory::primitive_desc pd = memory::primitive_desc(md, getEngine()); - mkldnn_primitive_t result; - mkldnn::error::wrap_c_api( - mkldnn_primitive_create(&result, pd.get(), nullptr, nullptr), - "could not create a memory primitive"); - reset(result); - set_data_handle(data_); + resetMKLDNNMemory(pd, data_); } } // namespace paddle diff --git a/paddle/math/MKLDNNMatrix.h b/paddle/math/MKLDNNMatrix.h index 5f5b819017b83579ce58522198b3f13311297d42..39d40a1f61609a649d3341c170d24b0604921ac2 100644 --- a/paddle/math/MKLDNNMatrix.h +++ b/paddle/math/MKLDNNMatrix.h @@ -102,6 +102,11 @@ public: m_->copyFrom(src); } + void copyTo(Matrix& dst) { + // TODO(TJ): reorder data if this format is not nchw or x + dst.copyFrom(*m_); + } + public: /** * Reorder this MKLDNNMatrix from other format. @@ -140,6 +145,27 @@ public: m_.reset(); } + /** + * override the CpuMatrix::resize + */ + void resize(size_t newHeight, size_t newWidth) override { + m_->resize(newHeight, newWidth); + if (data_ == m_->getData() && elementCnt_ == newHeight * newWidth) { + return; + } + CpuMatrix::setData(data_); + height_ = newHeight; + width_ = newWidth; + elementCnt_ = newHeight * newWidth; + stride_ = width_; + auto pd = mkldnn::memory::primitive_desc( + mkldnn::memory::desc({(int)newHeight, (int)newWidth}, + getDtype(), + mkldnn::memory::format::nc), + getEngine()); + resetMKLDNNMemory(pd, data_); + } + /** * override Matrix::getData * check data before return @@ -210,6 +236,17 @@ protected: memory::format srcFmt, memory::format dstFmt, memory::dims dm); + /** + * reset this MKLDNN Memory from primitve desc + */ + void resetMKLDNNMemory(memory::primitive_desc pd, real* data) { + mkldnn_primitive_t result; + mkldnn::error::wrap_c_api( + mkldnn_primitive_create(&result, pd.get(), nullptr, nullptr), + "could not create a memory primitive"); + reset(result); + set_data_handle(data); + } private: // save the CpuMatrixPtr in case the buffer released outside diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index c2f17beeb87942ea681f5d388659c0d280157b26..ba86eacbb5d53ee43a60d2cd1dd922333a5d48f0 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -206,7 +206,7 @@ double dotProduct(const int n, const double* x, const double* y) { } #endif -#if defined(PADDLE_USE_MKL) || defined(PADDLE_USE_MKLML) +#if defined(PADDLE_USE_MKLML) template <> void vExp(const int n, const float* a, float* r) { @@ -295,38 +295,6 @@ template void vAdd(const int n, const double* a, const double* b, double* r); #endif -#ifdef PADDLE_USE_MKL -template <> -void vInvSqrt(const int n, const float* a, float* r) { - vsInvSqrt(n, a, r); -} - -template <> -void vInvSqrt(const int n, const double* a, double* r) { - vdInvSqrt(n, a, r); -} - -template <> -void vLog1p(const int n, const float* a, float* r) { - vsLog1p(n, a, r); -} - -template <> -void vLog1p(const int n, const double* a, double* r) { - vdLog1p(n, a, r); -} - -template <> -void vTanh(const int n, const float* a, float* r) { - vsTanh(n, a, r); -} - -template <> -void vTanh(const int n, const double* a, double* r) { - vdTanh(n, a, r); -} -#else - DEFINE_MATRIX_BINARY_OP(vInvSqrt, b = 1.0f / std::sqrt(a)); template void vInvSqrt(const int n, const T* a, T* r) { @@ -357,6 +325,4 @@ template void vLog1p(const int n, const double* a, double* r); template void vTanh(const int n, const float* a, float* r); template void vTanh(const int n, const double* a, double* r); -#endif - } // namespace paddle diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index 8193aa4adffc0409d8ea68417c68fa153a2942d8..f6e77029bdd75a602f88b688ca810f47ba4ee615 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -21,11 +21,6 @@ limitations under the License. */ #include #endif -#ifdef PADDLE_USE_MKL -#include -#include -#endif - #if defined(PADDLE_USE_ATLAS) || defined(PADDLE_USE_VECLIB) extern "C" { #include diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index ff72672e3ab77212b309fcfea835839a916fa632..346008439c35a2bcbcd2e9dfd36d689e01d7495f 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -18,6 +18,7 @@ limitations under the License. */ #include #include "Matrix.h" #include "hl_gpu.h" +#include "hl_matrix.h" #include "hl_table_apply.h" #include "paddle/utils/Flags.h" #include "paddle/utils/Logging.h" @@ -99,6 +100,19 @@ MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { return mat; } +template <> +std::shared_ptr> VectorT::castToInt() { + std::shared_ptr> ret = IVector::create(this->getSize(), useGpu_); + if (useGpu_) { + hl_vector_cast2int(ret->getData(), this->getData(), this->getSize()); + } else { + for (size_t i = 0; i < getSize(); ++i) { + ret->getData()[i] = int(this->getData()[i]); + } + } + return ret; +} + template GpuVectorT::GpuVectorT(size_t size) : VectorT(size, diff --git a/paddle/math/Vector.h b/paddle/math/Vector.h index 80b9775fccf10c57bb48145ef56165ec7c86d8b8..f965a5809209da313c78a545c44e7aa39e95ac65 100644 --- a/paddle/math/Vector.h +++ b/paddle/math/Vector.h @@ -162,6 +162,13 @@ public: */ std::shared_ptr toOneHotSparseMatrix(size_t idRange, bool useGpu); + /** + * @brief cast vector of "real" elements to "int" elements. + * + * @note: float -> int must be casted, or you'll get wrong data. + */ + std::shared_ptr> castToInt(); + /** * This function will crash if the size of src and dest is different. */ diff --git a/paddle/math/tests/TensorCheck.h b/paddle/math/tests/TensorCheck.h index 5bc4a03067a75527fa30e5bb5526f93dc7b9fdcc..b998e5772e70d0a0ec79dc4064dcbaa2c302efd2 100644 --- a/paddle/math/tests/TensorCheck.h +++ b/paddle/math/tests/TensorCheck.h @@ -169,7 +169,7 @@ void TensorCheck(AssertEq compare, count++; } } - EXPECT_EQ(count, 0) << "There are " << count << " different element."; + EXPECT_EQ(count, 0) << "There are " << count << " different elements."; } template diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 60dc55a32f5f05875e4f3ce77431556e14adc74a..709f7de2e43093114d096cbfca5b5d49293a6d3e 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -62,6 +62,11 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_OP(pool2d);\n") endif() + if ("${TARGET}" STREQUAL "compare_op") + set(pybind_flag 1) + file(APPEND ${pybind_file} "USE_OP(less_than);\nUSE_OP(equal);\n") + endif() + # pool_with_index_op contains several operators if ("${TARGET}" STREQUAL "pool_with_index_op") set(pybind_flag 1) @@ -69,6 +74,20 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_OP(max_pool2d_with_index);\n") endif() + # conv_op contains several operators + if ("${TARGET}" STREQUAL "conv_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(conv2d);\n") + endif() + + # conv_transpose_op contains several operators + if ("${TARGET}" STREQUAL "conv_transpose_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(conv2d_transpose);\n") + endif() + # pool_cudnn_op contains several operators if ("${TARGET}" STREQUAL "pool_cudnn_op") set(pybind_flag 1) @@ -96,7 +115,7 @@ function(op_library TARGET) # It's enough to just adding one operator to pybind file(APPEND ${pybind_file} "USE_GPU_ONLY_OP(ncclAllReduce);\n") endif() - + # reduce_op contains several operators if ("${TARGET}" STREQUAL "reduce_op") set(pybind_flag 1) @@ -104,6 +123,11 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_OP(reduce_sum);\n") endif() + if ("${TARGET}" STREQUAL "tensor_array_read_write_op") + set(pybind_flag 1) + file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(read_from_array);\nUSE_NO_KERNEL_OP(write_to_array);\n") + endif() + # pybind USE_NO_KERNEL_OP # HACK: if REGISTER_OP_CPU_KERNEL presents the operator must have kernel file(READ ${TARGET}.cc TARGET_CONTENT) @@ -131,31 +155,54 @@ add_subdirectory(math) add_subdirectory(nccl) set(DEPS_OPS - recurrent_op cond_op cross_entropy_op + recurrent_op + dynamic_recurrent_op softmax_with_cross_entropy_op sum_op pool_op pool_with_index_op + conv_op + lstm_op + conv_transpose_op nccl_op sequence_conv_op - lstm_op) - + sequence_pool_op + lod_rank_table_op + lod_tensor_to_array_op + array_to_lod_tensor_op + lstm_op + tensor_array_read_write_op + gru_op) -op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc - DEPS framework_proto tensor net_op) op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op) op_library(cross_entropy_op DEPS cross_entropy) op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax) +op_library(conv_op DEPS vol2col) op_library(sum_op DEPS net_op selected_rows_functor) op_library(pool_op DEPS pooling) op_library(pool_with_index_op DEPS pooling) +op_library(lod_rank_table_op SRCS lod_rank_table_op.cc DEPS lod_rank_table) +op_library(lod_tensor_to_array_op SRCS lod_tensor_to_array_op.cc DEPS lod_rank_table_op) +op_library(array_to_lod_tensor_op SRCS array_to_lod_tensor_op.cc DEPS lod_rank_table_op) +op_library(tensor_array_read_write_op SRCS tensor_array_read_write_op.cc) if(WITH_GPU) op_library(nccl_op DEPS nccl_common) endif() op_library(sequence_conv_op DEPS context_project) +op_library(sequence_pool_op DEPS sequence_pooling) op_library(lstm_op DEPS sequence2batch lstm_compute) +op_library(conv_transpose_op DEPS vol2col) +op_library(gru_op DEPS sequence2batch gru_compute) +if(WITH_TESTING) + op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc + DEPS net_op tensor_array gtest) +else() + op_library(dynamic_recurrent_op SRCS dynamic_recurrent_op.cc rnn/recurrent_op_utils.cc + DEPS net_op tensor_array) +endif() +op_library(recurrent_op SRCS recurrent_op.cc DEPS executor) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) @@ -167,9 +214,11 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) +cc_test(beam_search_decode_op_test SRCS beam_search_decode_op_test.cc DEPS lod_tensor) cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) -cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc DEPS dynamic_recurrent_op recurrent_op tensor_array) - +cc_test(dynamic_recurrent_op_test SRCS dynamic_recurrent_op_test.cc + rnn/recurrent_op_utils.cc + DEPS dynamic_recurrent_op) if(WITH_GPU) nv_test(nccl_op_test SRCS nccl_op_test.cu DEPS nccl_op gpu_info device_context) endif() diff --git a/paddle/operators/accuracy_op.cc b/paddle/operators/accuracy_op.cc index 2a2a1e9cfd680ff983f54e4c12c34fbb5af69ca0..03c2fa945d94a522d25e65103c8842a93852ba3d 100644 --- a/paddle/operators/accuracy_op.cc +++ b/paddle/operators/accuracy_op.cc @@ -33,7 +33,7 @@ class AccuracyOp : public framework::OperatorWithKernel { auto inference_dim = ctx->GetInputDim("Out"); auto label_dim = ctx->GetInputDim("Label"); - // Assume indices has same shape with infernece, because + // Assume indices has same shape as inference, because // it's the output of topk. PADDLE_ENFORCE_EQ(label_dim.size(), 2, "label's rank must be 2."); @@ -47,10 +47,11 @@ class AccuracyOp : public framework::OperatorWithKernel { } protected: - // IndicateDataType - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext &ctx) const override { - return framework::ToDataType(ctx.Input("Out")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Out")->type()), + ctx.device_context()); } }; @@ -60,20 +61,24 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { // TODO(typhoonzero): support both inference value and indices. - AddInput("Out", "topk (inferences) the network output"); - AddInput("Indices", "topk (indices) the network output"); + AddInput("Out", "The network output of topk (inferences)"); + AddInput("Indices", "The the network output of topk (indices)"); AddInput("Label", "Label of the training data"); // TODO(typhoonzero): AddInput("Weight", ... AddOutput("Accuracy", "The accuracy of current batch"); AddComment(R"DOC( -Accuracy. It will print accuracy rate for classification. -The accuracy is: -.. math:: -accuracy = \\frac{NumOfCorrectPredicts}{NumOfAllSamples}) +Accuracy Operator. + +It will print accuracy rate for classification. +The accuracy is calculated as follows: + +$$accuracy = \frac{NumOfCorrectPredicts}{NumOfAllSamples}$$ + +Both the input Out and Label can carry the LoD (Level of Details) +information, or not. But the output only shares the LoD information +with the input Out(Inference). -Both the input `Out` and `Label` can carry the LoD (Level of Details) -information, or not. But the output only shares the LoD with input `Inference`. )DOC"); } }; diff --git a/paddle/operators/accuracy_op.cu b/paddle/operators/accuracy_op.cu index a0483f367e1e3bcf4ad50ee9155f25461c1b4ed2..1776f33105367447759aa91c25263dfc53bd2f99 100644 --- a/paddle/operators/accuracy_op.cu +++ b/paddle/operators/accuracy_op.cu @@ -65,18 +65,15 @@ class AccuracyOpCUDAKernel : public framework::OpKernel { size_t num_samples = inference->dims()[0]; size_t infer_width = inference->dims()[1]; - cudaMemset((void**)&accuracy_data, 0, sizeof(float)); + PADDLE_ENFORCE(cudaMemset(accuracy_data, 0, sizeof(float))); if (num_samples == 0) { return; } AccuracyCudaKernel<<< - 1, PADDLE_CUDA_NUM_THREADS, 0, - reinterpret_cast( - ctx.device_context()) - .stream()>>>(num_samples, infer_width, indices_data, label_data, - accuracy_data); + 1, PADDLE_CUDA_NUM_THREADS, 0, ctx.cuda_device_context().stream()>>>( + num_samples, infer_width, indices_data, label_data, accuracy_data); } }; diff --git a/paddle/operators/accuracy_op.h b/paddle/operators/accuracy_op.h index 1968b53d19acfddaa514eca6e24d98a298d8d311..28dbc77f64842a62e88ae8df4ead7adc3b03764b 100644 --- a/paddle/operators/accuracy_op.h +++ b/paddle/operators/accuracy_op.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once #include -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" namespace paddle { @@ -22,18 +21,6 @@ namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; - -template -using EigenVector = framework::EigenVector; - -template -using EigenScalar = framework::EigenScalar; - template class AccuracyKernel : public framework::OpKernel { public: diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 90f1535fcd387c34ea39d84d9c2ec78fcbc3c764..83d35a450d0e8ebf5311cdfd948b066642ccec8c 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -43,7 +43,12 @@ class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Sigmoid operator"); AddOutput("Y", "Output of Sigmoid operator"); - AddComment("Sigmoid activation operator, sigmoid = 1 / (1 + exp(-x))"); + AddComment(R"DOC( +Sigmoid Activation Operator. + +$y = 1 / (1 + e^{-x})$ + +)DOC"); } }; @@ -54,8 +59,12 @@ class LogSigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of LogSigmoid operator"); AddOutput("Y", "Output of LogSigmoid operator"); - AddComment( - "Logsigmoid activation operator, logsigmoid = log (1 / (1 + exp(-x)))"); + AddComment(R"DOC( +Logsigmoid Activation Operator. + +$y = \log(1 / (1 + e^{-x}))$ + +)DOC"); } }; @@ -65,7 +74,12 @@ class ExpOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Exp operator"); AddOutput("Y", "Output of Exp operator"); - AddComment("Exp activation operator, exp(x) = e^x"); + AddComment(R"DOC( +Exp Activation Operator. + +$y = e^x$ + +)DOC"); } }; @@ -75,7 +89,12 @@ class ReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Relu operator"); AddOutput("Y", "Output of Relu operator"); - AddComment("Relu activation operator, relu(x) = max(x, 0)"); + AddComment(R"DOC( +Relu Activation Operator. + +$y = \max(x, 0)$ + +)DOC"); } }; @@ -87,11 +106,14 @@ class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of LeakyRelu operator"); AddOutput("Y", "Output of LeakyRelu operator"); - AddComment( - "LeakyRelu activation operator, " - "leaky_relu = max(x, alpha * x)"); AddAttr("alpha", "The small negative slope") .SetDefault(static_cast(0.02f)); + AddComment(R"DOC( +LeakyRelu Activation Operator. + +$y = \max(x, \alpha * x)$ + +)DOC"); } }; @@ -103,12 +125,20 @@ class SoftShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softshrink operator"); AddOutput("Y", "Output of Softshrink operator"); - AddComment( - "Softshrink activation operator, " - "softshrink = x - lambda, if x > lambda;" - " x + lambda, if x < lambda; 0 otherwise"); AddAttr("lambda", "non-negative offset") .SetDefault(static_cast(0.5f)); + AddComment(R"DOC( +Softshrink Activation Operator. + +$$ +y = \begin{cases} + x - \lambda, \text{if } x > \lambda \\ + x + \lambda, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -118,9 +148,12 @@ class TanhOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Tanh operator"); AddOutput("Y", "Output of Tanh operator"); - AddComment( - "Tanh activation operator, tanh = (exp(x) - exp(-x)) / (exp(x) + " - "exp(-x))"); + AddComment(R"DOC( +Tanh Activation Operator. + +$$y = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ + +)DOC"); } }; @@ -131,7 +164,12 @@ class TanhShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of TanhShrink operator"); AddOutput("Y", "Output of TanhShrink operator"); - AddComment("TanhShrink activation operator, tanhshrink(x) = x - tanh(x)"); + AddComment(R"DOC( +TanhShrink Activation Operator. + +$$y = x - \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}$$ + +)DOC"); } }; @@ -143,13 +181,20 @@ class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of HardShrink operator"); AddOutput("Y", "Output of HardShrink operator"); - AddComment( - "HardShrink activation operator, " - "hard_shrink(x) = x if x > lambda" - "hard_shrink(x) = x if x < -lambda" - "hard_shrink(x) = 0 otherwise"); AddAttr("threshold", "The value of threshold for HardShrink") .SetDefault(static_cast(0.5)); + AddComment(R"DOC( +HardShrink Activation Operator. + +$$ +y = \begin{cases} + x, \text{if } x > \lambda \\ + x, \text{if } x < -\lambda \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -159,7 +204,12 @@ class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Sqrt operator"); AddOutput("Y", "Output of Sqrt operator"); - AddComment("Sqrt activation operator, sqrt(x) = x^(1/2)"); + AddComment(R"DOC( +Sqrt Activation Operator. + +$y = \sqrt{x}$ + +)DOC"); } }; @@ -169,7 +219,12 @@ class AbsOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Abs operator"); AddOutput("Y", "Output of Abs operator"); - AddComment("Abs activation operator, abs(x) = |x|"); + AddComment(R"DOC( +Abs Activation Operator. + +$y = |x|$ + +)DOC"); } }; @@ -180,7 +235,12 @@ class ReciprocalOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Reciprocal operator"); AddOutput("Y", "Output of Reciprocal operator"); - AddComment("Reciprocal activation operator, reciprocal(x) = 1 / x"); + AddComment(R"DOC( +Reciprocal Activation Operator. + +$$y = \frac{1}{x}$$ + +)DOC"); } }; @@ -190,7 +250,14 @@ class LogOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Log operator"); AddOutput("Y", "Output of Log operator"); - AddComment("Log activation operator, log(x) = natural logarithm of x"); + AddComment(R"DOC( +Log Activation Operator. + +$y = \ln(x)$ + +Natural logarithm of x. + +)DOC"); } }; @@ -200,7 +267,12 @@ class SquareOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Square operator"); AddOutput("Y", "Output of Square operator"); - AddComment("Square activation operator, square(x) = x^2"); + AddComment(R"DOC( +Square Activation Operator. + +$y = x^2$ + +)DOC"); } }; @@ -211,7 +283,12 @@ class SoftplusOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softplus operator"); AddOutput("Y", "Output of Softplus operator"); - AddComment("Softplus activation operator, softplus(x) = log(1 + exp(x))"); + AddComment(R"DOC( +Softplus Activation Operator. + +$y = \ln(1 + e^{x})$ + +)DOC"); } }; @@ -222,7 +299,12 @@ class SoftsignOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Softsign operator"); AddOutput("Y", "Output of Softsign operator"); - AddComment("Softsign activation operator, softsign(x) = x / (1 + |x|)"); + AddComment(R"DOC( +Softsign Activation Operator. + +$$y = \frac{x}{1 + |x|}$$ + +)DOC"); } }; @@ -233,11 +315,16 @@ class BReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of BRelu operator"); AddOutput("Y", "Output of BRelu operator"); - AddComment("BRelu activation operator, brelu = max(min(x, t_min), t_max)"); AddAttr("t_min", "The min marginal value of BRelu") .SetDefault(static_cast(0)); AddAttr("t_max", "The max marginal value of BRelu") .SetDefault(static_cast(24)); + AddComment(R"DOC( +BRelu Activation Operator. + +$y = \max(\min(x, t_{min}), t_{max})$ + +)DOC"); } }; @@ -249,11 +336,14 @@ class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of SoftRelu operator"); AddOutput("Y", "Output of SoftRelu operator"); - AddComment( - "SoftRelu activation operator, soft_relu = log(1 + exp(max(min(x, " - "threshold), threshold)))"); AddAttr("threshold", "The threshold value of SoftRelu") .SetDefault(static_cast(40)); + AddComment(R"DOC( +SoftRelu Activation Operator. + +$y = \ln(1 + \exp(\max(\min(x, threshold), threshold))$ + +)DOC"); } }; @@ -262,19 +352,19 @@ class ELUOpMaker : public framework::OpProtoAndCheckerMaker { public: ELUOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", - "(Tensor) The input of ELU operator, it shouldn't be empty. Input " - "is flattened and treated as a 1D array."); - AddOutput("Y", - "(Tensor) The output of ELU operator. It has the same shape as " - "the input."); - AddAttr( - "alpha", "(float, default 1.0) Alpha value in the elu formulation.") - .SetDefault(static_cast(1.)); + AddInput("X", "Input of ELU operator"); + AddOutput("Y", "Output of ELU operator"); + AddAttr("alpha", "The alpha value of ELU") + .SetDefault(static_cast(1.0f)); AddComment(R"DOC( - ELU activation operator. It applies this element-wise computation on - the input: f(x) = max(0, x) + min(0, alpha * (exp(x) - 1)). - Check .. _Link: https://arxiv.org/abs/1511.07289 for more details.)DOC"); +ELU Activation Operator. + +Applies the following element-wise computation on the input according to +https://arxiv.org/abs/1511.07289. + +$y = \max(0, x) + \min(0, \alpha * (e^x - 1))$ + +)DOC"); } }; @@ -285,9 +375,14 @@ class Relu6OpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Relu6 operator"); AddOutput("Y", "Output of Relu6 operator"); - AddComment("Relu6 activation operator, relu6 = min(max(0, x), 6)"); AddAttr("threshold", "The threshold value of Relu6") .SetDefault(static_cast(6)); + AddComment(R"DOC( +Relu6 Activation Operator. + +$y = \min(\max(0, x), 6)$ + +)DOC"); } }; @@ -298,9 +393,14 @@ class PowOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of Pow operator"); AddOutput("Y", "Output of Pow operator"); - AddComment("Pow activation operator, pow(x, factor) = x^factor"); AddAttr("factor", "The exponential factor of Pow") .SetDefault(static_cast(1)); + AddComment(R"DOC( +Pow Activation Operator. + +$y = x^{factor}$ + +)DOC"); } }; @@ -311,11 +411,16 @@ class STanhOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of STanh operator"); AddOutput("Y", "Output of STanh operator"); - AddComment("STanh activation operator, stanh = b * tanh(a * x)"); AddAttr("scale_a", "The scale parameter of a for the input") .SetDefault(static_cast(2 / 3)); AddAttr("scale_b", "The scale parameter of b for the input") .SetDefault(static_cast(1.7159)); + AddComment(R"DOC( +STanh Activation Operator. + +$$y = b * \frac{e^{a * x} - e^{-a * x}}{e^{a * x} + e^{-a * x}}$$ + +)DOC"); } }; @@ -327,12 +432,19 @@ class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of ThresholdedRelu operator"); AddOutput("Y", "Output of ThresholdedRelu operator"); - AddComment( - "ThresholdedRelu activation operator, " - "thresholded_relu = x for x > threshold, " - "thresholded_relu = 0 otherwise."); AddAttr("threshold", "The threshold location of activation") .SetDefault(static_cast(1.0)); + AddComment(R"DOC( +ThresholdedRelu Activation Operator. + +$$ +y = \begin{cases} + x, \text{if } x > threshold \\ + 0, \text{otherwise} + \end{cases} +$$ + +)DOC"); } }; @@ -344,27 +456,23 @@ class HardSigmoidOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "Input of HardSigmoid operator"); AddOutput("Y", "Output of HardSigmoid operator"); + AddAttr("slope", "Slope for linear approximation of sigmoid") + .SetDefault(static_cast(0.2)); + AddAttr("offset", "Offset for linear approximation of sigmoid") + .SetDefault(static_cast(0.5)); AddComment(R"DOC( -Hard Sigmoid activation operator. +HardSigmoid Activation Operator. -Segment-wise linear approximation of sigmoid[1]. -This is much faster than sigmoid. +Segment-wise linear approximation of sigmoid(https://arxiv.org/abs/1603.00391), +which is much faster than sigmoid. -hard_sigmoid = max(0, min(1, slope * x + shift)) +$y = \max(0, \min(1, slope * x + shift))$ The slope should be positive. The offset can be either positive or negative. -The default slope and shift are set from [1]. +The default slope and shift are set according to the above reference. It is recommended to use the defaults for this activation. -References: - [1] Noisy Activation Functions - (https://arxiv.org/abs/1603.00391) - - )DOC"); - AddAttr("slope", "Slope for linear approximation of sigmoid") - .SetDefault(static_cast(0.2)); - AddAttr("offset", "Offset for linear approximation of sigmoid") - .SetDefault(static_cast(0.5)); +)DOC"); } }; diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index ddd966e26c9abad0d83f8b5c6e3e7d9ad65158a8..ceb4b4e40b67473f42e67e3f02f8e012e1b1eb50 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -232,7 +232,7 @@ struct HardShrinkGradFunctor : public BaseActivationFunctor { } }; -// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < lambda; 0 +// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < -lambda; 0 // otherwise template struct SoftShrinkFunctor : public BaseActivationFunctor { diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc index 24e419b532d97bc16ab96dad418d6e73c03f30a0..b717e1647e4b89285b841420650dc69e8a1e0c58 100644 --- a/paddle/operators/adadelta_op.cc +++ b/paddle/operators/adadelta_op.cc @@ -64,16 +64,15 @@ class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Param", "(Tensor) Input parameter"); AddInput("Grad", "(Tensor) Input gradient"); - AddInput("AvgSquaredGrad", - "(Tensor) Input expectation of squared gradient"); + AddInput("AvgSquaredGrad", "(Tensor) Input average of squared gradient"); AddInput("AvgSquaredUpdate", - "(Tensor) Input expectation of squared parameter updates"); + "(Tensor) Input average of squared parameter updates"); AddOutput("ParamOut", "(Tensor) Output parameter"); AddOutput("AvgSquaredGradOut", - "(Tensor) Output expectation of squared gradient"); + "(Tensor) Output average of squared gradient"); AddOutput("AvgSquaredUpdateOut", - "(Tensor) Output expectation of squared parameter updates"); + "(Tensor) Output average of squared parameter updates"); AddAttr("rho", "(float, default 0.95) Exponential decay rate " @@ -84,22 +83,21 @@ class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker { "numerical stability") .SetDefault(1.0e-6f); AddComment(R"DOC( -Adadelta Updates Operator. +Adadelta Optimizer. -This implements the Adadelta optimizer[1]. Adadelta is a per-dimension -adaptive learning rate method for gradient descent. +Adadelta optimizer is implemented as explained in: +https://arxiv.org/abs/1212.5701 +Adadelta is a per-dimension adaptive learning rate method used +for gradient descent. -Adadelta updates: +Adadelta updates are as follows: -avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * grad * grad -param_update = - sqrt((avg_squared_update + epsilon) / - (avg_squared_grad_out + epsilon)) * grad -avg_squared_update_out = rho * avg_squared_update + (1 - rho) * param_update**2 -param_out = param + param_update - -References: - [1] ADADELTA: An Adaptive Learning Rate Method - https://arxiv.org/abs/1212.5701 +$$avgSquaredGradOut = \rho * avgSquaredGrad + (1 - \rho) * grad * grad \break +paramUpdate = - $\sqrt{((avgSquaredUpdate + \epsilon) / + (avgSquaredGrad_out + \epsilon))}$ * grad \break +avgSquaredUpdateOut = \rho * avgSquaredUpdate + (1 - \rho) * + {(paramUpdate)}^2 \break +paramOut = param + paramUpdate$$ )DOC"); } diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index bc081f87dcab0dcd8ef329dcb1f66b627c82b4a2..8d1a2b7938d2c6607cbeb3cecb72d1d5b83dd8b9 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -73,12 +73,16 @@ class AdagradOpMaker : public framework::OpProtoAndCheckerMaker { Adaptive Gradient Algorithm (Adagrad). -moment_out = moment + grad * grad -param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon) +The update is done as follows: + +$$momentOut = moment + grad * grad \break +paramOut = param - learningRate * grad / ($\sqrt{momentOut}$ + \epsilon) \break +$$ The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) -does not have the epsilon attribute. It is added here for numerical stability -by avoiding division by zero. +does not have the epsilon attribute. It is added here in our implementation +as also proposed here: http://cs231n.github.io/neural-networks-3/#ada +for numerical stability to avoid the division by zero error. )DOC"); } diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc index 3572de06bd60f7979e3bfbf39856b04942ce81c0..97a091ae766abfba5412bbd32c34a6f80701fbf7 100644 --- a/paddle/operators/adam_op.cc +++ b/paddle/operators/adam_op.cc @@ -51,8 +51,8 @@ class AdamOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1, "Beta1 power accumulator should have 1 dimension"); auto beta2_pow_dims = ctx->GetInputDim("Beta2Pow"); - PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1, - "Beta1 power accumulator should have 1 dimension"); + PADDLE_ENFORCE_EQ(framework::product(beta2_pow_dims), 1, + "Beta2 power accumulator should have 1 dimension"); auto param_dims = ctx->GetInputDim("Param"); PADDLE_ENFORCE_EQ( @@ -60,10 +60,10 @@ class AdamOp : public framework::OperatorWithKernel { "Param and Grad input of AdamOp should have same dimension"); PADDLE_ENFORCE_EQ( param_dims, ctx->GetInputDim("Moment1"), - "Param and Moment input of AdamOp should have same dimension"); + "Param and Moment1 input of AdamOp should have same dimension"); PADDLE_ENFORCE_EQ( param_dims, ctx->GetInputDim("Moment2"), - "Param and InfNorm input of AdamOp should have same dimension"); + "Param and Moment2 input of AdamOp should have same dimension"); ctx->SetOutputDim("ParamOut", param_dims); ctx->SetOutputDim("Moment1Out", param_dims); @@ -103,23 +103,20 @@ class AdamOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(1.0e-8f); AddComment(R"DOC( -Adam Updates Operator. +Adam Optimizer. This implements the Adam optimizer from Section 2 of the Adam -paper[1]. Adam is a first-order gradient-based optimization -method based on adaptive estimates of lower-order moments. +paper : https://arxiv.org/abs/1412.6980. +Adam is a first-order gradient-based optimization method based on +adaptive estimates of lower-order moments. Adam updates: -moment1_out = beta1 * moment1 + (1 − beta1) * grad -moment2_out = beta2 * moment2 + (1 − beta2) * grad * grad -learning_rate_t = learning_rate_t * - sqrt(1 - beta2_pow) / (1 - beta1_pow) -param_out = param - learning_rate_t * moment1/ (sqrt(moment2) + epsilon) - -References: - [1] Adam: A Method for Stochastic Optimization - (https://arxiv.org/abs/1412.6980) +$$moment_1_{out} = \beta_1 * moment_1 + (1 - \beta_1) * grad \break +moment_2_{out} = \beta_2 * moment_2 + (1 - \beta_2) * grad * grad \break +learningRate = learningRate * + $\sqrt{(1 - \beta_2_{pow})}$ / (1 - \beta_1_{pow}) \break +paramOut = param - learningRate * moment_1/ ($\sqrt{(moment_2)} + \epsilon)$$ )DOC"); } diff --git a/paddle/operators/adamax_op.cc b/paddle/operators/adamax_op.cc index ff2565774115571166712b03c8990e5bf8de12a5..14cf3841b33a8153549e4c99ed2b75286e9c64db 100644 --- a/paddle/operators/adamax_op.cc +++ b/paddle/operators/adamax_op.cc @@ -99,26 +99,22 @@ class AdamaxOpMaker : public framework::OpProtoAndCheckerMaker { "Constant for numerical stability") .SetDefault(1.0e-8f); AddComment(R"DOC( -Adamax Updates Operator. +Adamax Optimizer. -This implements the Adamax optimizer from Section 7 of the Adam -paper[1]. Adamax is a variant of the +We implement the Adamax optimizer from Section 7 of the Adam +paper: https://arxiv.org/abs/1412.6980. Adamax is a variant of the Adam algorithm based on the infinity norm. Adamax updates: -moment_out = beta1 * moment + (1 - beta1) * grad -inf_norm_out = max(beta2 * inf_norm + epsilon, abs(grad)) -learning_rate_t = learning_rate/(1 - beta1_pow) -param_out = param - learning_rate_t * moment_out/inf_norm_out +$$momentOut = \beta_1 * moment + (1 - \beta_1) * grad \break +infNormOut = max(\beta_2 * infNorm + \epsilon, |grad|) \break +learningRate = learningRate /(1 - \beta_1_{pow}) \break +paramOut = param - learningRate * momentPut / infNormOut$$ The original paper does not have an epsilon attribute. -However, it is added here for numerical stability -by preventing divide by 0. - -References: - [1] Adam: A Method for Stochastic Optimization - (https://arxiv.org/abs/1412.6980) +However, it is added here for numerical stability to prevent the +division by 0 error. )DOC"); } diff --git a/paddle/operators/array_operator.h b/paddle/operators/array_operator.h new file mode 100644 index 0000000000000000000000000000000000000000..666043e824f885e9c0e79e319d0a38ba108c209a --- /dev/null +++ b/paddle/operators/array_operator.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { +class ArrayOp : public framework::OperatorBase { + public: + ArrayOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + size_t GetOffset(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const { + auto *i = scope.FindVar(Input("I")); + PADDLE_ENFORCE(i != nullptr, "I must be set"); + auto &i_tensor = i->Get(); + PADDLE_ENFORCE_EQ(i_tensor.numel(), 1); + size_t offset; + if (platform::is_gpu_place(i_tensor.place())) { + // FIXME: Avoid copy from GPU to CPU + framework::Tensor t; + t.CopyFrom(i_tensor, platform::CPUPlace(), dev_ctx); + dev_ctx.Wait(); + offset = static_cast(*t.data()); + } else { + offset = static_cast(*i_tensor.data()); + } + return offset; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/array_to_lod_tensor_op.cc b/paddle/operators/array_to_lod_tensor_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..c0903bb4e5ca7f160e19eefab99af7e3e4a8ed76 --- /dev/null +++ b/paddle/operators/array_to_lod_tensor_op.cc @@ -0,0 +1,170 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memcpy.h" + +namespace paddle { +namespace operators { + +using LoD = framework::LoD; + +class ArrayToLoDTensorOp : public framework::OperatorBase { + public: + ArrayToLoDTensorOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &rank_table = + scope.FindVar(Input("RankTable"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + + // Check dims, place and data type of input's elements and infer output's + // dim + PADDLE_ENFORCE(!x.empty(), "There's no element in the input array."); + int rank = x[0].dims().size(); + platform::Place place = x[0].place(); + std::type_index data_type = x[0].type(); + framework::DDim ins_dims = framework::slice_ddim(x[0].dims(), 1, rank); + int64_t batch_size = x[0].dims()[0]; + for (size_t i = 1; i < x.size(); ++i) { + PADDLE_ENFORCE_EQ(framework::slice_ddim(x[i].dims(), 1, rank), ins_dims, + "The dimension of the %zu'th element in LoDTensorArray " + "differs from previous ones.", + i); + PADDLE_ENFORCE(platform::places_are_same_class(x[i].place(), place), + "The place class of the %zu'th element in LoDTensorArray " + "differs from previous ones.", + i); + PADDLE_ENFORCE(x[i].type() == data_type, + "The date type of the %zu'th element in LoDTensorArray " + "differs from previous ones.", + i); + batch_size += x[i].dims()[0]; + } + auto ins_dim_vec = framework::vectorize(ins_dims); + ins_dim_vec.insert(ins_dim_vec.begin(), batch_size); + framework::DDim out_dims = framework::make_ddim(ins_dim_vec); + out->Resize(out_dims); + out->mutable_data(place, data_type); + + auto &table_items = rank_table.items(); + std::vector table_item_idx(table_items.size()); + // table_item_idx = range(table_items_idx.size()) + std::iota(table_item_idx.begin(), table_item_idx.end(), 0); + std::sort(table_item_idx.begin(), table_item_idx.end(), + [&](size_t a, size_t b) { + return table_items[a].index < table_items[b].index; + }); + + // Build LoDTensor `out` + framework::LoD *out_lod = out->mutable_lod(); + out_lod->clear(); + size_t out_offset = 0; + auto prefix_lod = rank_table.coarse_lod(); + prefix_lod.emplace_back(); + auto &cur_level_lod = prefix_lod.back(); + cur_level_lod.push_back(0); + for (size_t idx : table_item_idx) { + cur_level_lod.push_back(cur_level_lod.back() + table_items[idx].length); + for (size_t x_idx = 0; x_idx < table_items[idx].length; ++x_idx) { + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + x[x_idx].lod(), idx, idx + 1, 0); + + auto &lod_length = lod_and_offset.first; + framework::AppendLoD(out_lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + VLOG(10) << "idx=" << idx << " x_idx=" << x_idx << " [" + << ", " << end_offset << "]"; + // Copy data + PADDLE_ENFORCE_GE(end_offset, start_offset); + size_t len = end_offset - start_offset; + if (len == 0) { + continue; + } + out->Slice(out_offset, out_offset + len) + .CopyFrom(x[x_idx].Slice(start_offset, end_offset), place, dev_ctx); + out_offset += len; + } + } + out_lod->insert(out_lod->begin(), prefix_lod.begin(), prefix_lod.end()); + } +}; + +class ArrayToLoDTensorOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ArrayToLoDTensorOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(std::vector) A vector of tensors that is going to " + "be casted to a big LoDTensor."); + AddInput("RankTable", + "(LoDRankTable) RankTable provides the coarse lod infomation to " + "build the output LoDTensor. See " + "'paddle/framework/lod_rank_table.h' for more details."); + AddOutput("Out", "(LoDTensor) The LoDTensor formed by input tensor array."); + AddComment( + R"DOC(This Op build a big LoDTensor from a std::vector + and a LoDRankTable. It is supposed to be used in getting dynamic RNN's + outputs back to a normal LoDTensor. The std::vector + would be the output of RNN Op and the LoDRankTable would be build + with RNN's input.)DOC"); + } +}; + +class ArrayToLoDTensorInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "ArrayToLoDTensorOp must has input X."); + PADDLE_ENFORCE(context->HasInput("RankTable"), + "ArrayToLoDTensorOp must has input RankTable."); + context->SetOutputDim("Out", context->GetInputDim("X")); + } +}; + +class ArrayToLoDTensorGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("lod_tensor_to_array"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("RankTable", Input("RankTable")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(array_to_lod_tensor, ops::ArrayToLoDTensorOp, + ops::ArrayToLoDTensorOpProtoMaker, + ops::ArrayToLoDTensorInferShape, + ops::ArrayToLoDTensorGradMaker); diff --git a/paddle/operators/assign_op.cc b/paddle/operators/assign_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..609e915b932e2bc4d5abee1e5f868cc07a7619d3 --- /dev/null +++ b/paddle/operators/assign_op.cc @@ -0,0 +1,138 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/data_type.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/var_type.h" + +namespace paddle { +namespace operators { +class AssignFunctor { + public: + AssignFunctor(framework::Variable *out, + const platform::DeviceContext &dev_ctx) + : out_(out), dev_ctx_(dev_ctx) {} + + void operator()(const framework::LoDTensor &lod_tensor) const { + auto &out_tensor = *out_->GetMutable(); + copy_tensor(lod_tensor, &out_tensor); + } + + void operator()(const framework::LoDTensorArray &array) const { + auto &out_array = *out_->GetMutable(); + out_array.resize(array.size()); + for (size_t i = 0; i < array.size(); ++i) { + copy_tensor(array[i], &out_array[i]); + } + } + + void operator()(const framework::SelectedRows &rows) const { + framework::SelectedRows &out_rows = + *out_->GetMutable(); + out_rows.set_rows(rows.rows()); + out_rows.set_height(rows.height()); + auto &t = rows.value(); + out_rows.mutable_value()->CopyFrom(t, t.place(), dev_ctx_); + } + + template + void operator()(const T &v) const { + PADDLE_THROW("Not support type for assign op %s", typeid(T).name()); + } + + private: + void copy_tensor(const framework::LoDTensor &lod_tensor, + framework::LoDTensor *out) const { + auto &out_tensor = *out; + out_tensor.CopyFrom(lod_tensor, lod_tensor.place(), dev_ctx_); + out_tensor.set_lod(lod_tensor.lod()); + } + + framework::Variable *out_; + const platform::DeviceContext &dev_ctx_; +}; + +class AssignOp : public framework::OperatorBase { + public: + AssignOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x = scope.FindVar(Input("X")); + if (x == nullptr) { + return; + } + auto *out = scope.FindVar(Output("Out")); + PADDLE_ENFORCE( + out != nullptr, + "The Output(Out) should not be null if the Input(X) is set."); + framework::VisitVarType(*x, AssignFunctor(out, dev_ctx)); + } +}; + +class AssignOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + AssignOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor, SelectedRows or LoDTensorArray) The input variable " + "could be LoDTensor, SelectedRows or LoDTensorArray.") + .AsDispensable(); + AddOutput("Out", + "(LoDTensor, SelectedRows or LoDTensorArray) The type of output " + "is the same as input X."); + AddComment(R"DOC(Assign Operator + +Out = X, when type in [LoDTensor/SelectedRows/LoDTensorArray] +raise error if the type is not listed above. +)DOC"); + } +}; + +class AssignInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + if (context->HasInput("X")) { + auto type = context->GetInputsVarType("X")[0]; + if (type == framework::VarDesc_VarType_SELECTED_ROWS || + type == framework::VarDesc_VarType_LOD_TENSOR) { + context->SetOutputDim("Out", context->GetInputDim("X")); + } + } + } +}; + +class AssignGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *op = new framework::OpDescBind(); + op->SetType("assign"); + op->SetInput("X", OutputGrad("Out")); + op->SetOutput("Out", InputGrad("X")); + return std::unique_ptr(op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(assign, ops::AssignOp, ops::AssignGradMaker, + ops::AssignInferShape, ops::AssignOpProtoMaker); diff --git a/paddle/operators/auc_op.cc b/paddle/operators/auc_op.cc index f5784922af1cafcf4e6873c0e906bbd66ce8ae69..6c3f67ec32fb1b942241997e87a1e9c4752e707d 100644 --- a/paddle/operators/auc_op.cc +++ b/paddle/operators/auc_op.cc @@ -23,11 +23,11 @@ class AucOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContext *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("Out"), "Input of Out must be initialized."); + PADDLE_ENFORCE(ctx->HasInput("Out"), "Input of Out should not be null."); PADDLE_ENFORCE(ctx->HasInput("Indices"), - "Input of Indices must be initialized."); + "Input of Indices should not be null."); PADDLE_ENFORCE(ctx->HasInput("Label"), - "Input of Label must be initialized."); + "Input of Label should not be null."); auto inference_height = ctx->GetInputDim("Out")[0]; auto label_height = ctx->GetInputDim("Label")[0]; @@ -39,10 +39,11 @@ class AucOp : public framework::OperatorWithKernel { } protected: - // IndicateDataType - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext &ctx) const override { - return framework::ToDataType(ctx.Input("Out")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Out")->type()), + ctx.device_context()); } }; @@ -52,20 +53,20 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Out", "A floating point 2D tensor, values are in the range [0, 1]." - "Each row is descend sorted. This input should be the" + "Each row is sorted in descending order. This input should be the" "output of topk." "Typically, this tensor indicates the probability of each label"); AddInput("Indices", "An int 2D tensor, indicating the indices of original" - "tensor before sort. Typically, this tensor indicates which label" - "the probability stands for."); + "tensor before sorting. Typically, this tensor indicates which " + "label the probability stands for."); AddInput("Label", "A 2D int tensor indicating the label of the training data." "The height is batch size and width is always 1."); // TODO(typhoonzero): support weight input AddOutput("AUC", "A scalar representing the " - "current area-under-curve."); + "current area-under-the-curve."); AddAttr("curve", "Curve type, can be 'ROC' or 'PR'.") .SetDefault("ROC"); @@ -74,19 +75,18 @@ class AucOpMaker : public framework::OpProtoAndCheckerMaker { " roc curve.") .SetDefault(200); - AddComment( - R"DOC(Computes the AUC according forward output and label. -Best to use for binary classification evaluations. + AddComment(R"DOC( +Area Under The Curve (AUC) Operator. +This implementation computes the AUC according to forward output and label. +It is used very widely in binary classification evaluation. As a note: If input label contains values other than 0 and 1, it will be cast -to bool. - -You can find the definations here: +to bool. You can find the relevant definitions here: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve -Possible curves are: -- ROC: Receiver operating characteristic -- PR: Precision Recall +There are two types of possible curves: +1. ROC: Receiver operating characteristic +2. PR: Precision Recall )DOC"); } }; diff --git a/paddle/operators/batch_norm_op.cc b/paddle/operators/batch_norm_op.cc index f2c8be4c54eed9cd0aeb004eeb74a42adc0695f5..f884e6efa917ce3f8554dce0e248f2b29273e3f3 100644 --- a/paddle/operators/batch_norm_op.cc +++ b/paddle/operators/batch_norm_op.cc @@ -19,9 +19,6 @@ namespace operators { using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -template -using EigenMatrix = framework::EigenMatrix; template using EigenArrayMap = @@ -51,6 +48,10 @@ class BatchNormOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("SavedMean"), ""); PADDLE_ENFORCE(ctx->HasOutput("SavedVariance"), ""); + const float epsilon = ctx->Attrs().Get("epsilon"); + PADDLE_ENFORCE_GE(epsilon, 0.0, "epsilon should be larger than 0"); + PADDLE_ENFORCE_LE(epsilon, 0.001, "epsilon should not be too large"); + // make sure Mean/MeanOut and Variance/VarianceOut share memory in Python PADDLE_ENFORCE_EQ(ctx->Inputs("Mean")[0], ctx->Outputs("MeanOut")[0], "Mean and MeanOut should share the same memory"); @@ -66,7 +67,7 @@ class BatchNormOp : public framework::OperatorWithKernel { : x_dims[x_dims.size() - 1]); PADDLE_ENFORCE(x_dims.size() >= 3 && x_dims.size() <= 5, - "Input x must have 3 to 5 dimensions."); + "Input X must have 3 to 5 dimensions."); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale").size(), 1UL); PADDLE_ENFORCE_EQ(ctx->GetInputDim("Scale")[0], C); @@ -93,16 +94,16 @@ class BatchNormOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "The input tensor"); AddInput("Scale", "Scale is a 1-dimensional tensor of size C " - "to be applied to the output"); + "that is applied to the output"); AddInput("Bias", "Bias is a 1-dimensional tensor of size C " - "to be applied to the output"); + "that is applied to the output"); AddInput("Mean", - "The global mean (for training) or the " + "The global mean (for training) or " "estimated mean (for testing)"); AddInput("Variance", "The global variance (for training) " - "or the estimated Variance (for testing)"); + "or estimated Variance (for testing)"); AddOutput("Y", "result after normalization"); AddOutput("MeanOut", "Share memory with Mean. " @@ -119,10 +120,14 @@ class BatchNormOpMaker : public framework::OpProtoAndCheckerMaker { "will apply to output when training") .AsIntermediate(); AddComment(R"DOC( -https://arxiv.org/pdf/1502.03167.pdf +Batch Normalization. -NHWC `[batch, in_height, in_width, in_channels]` -NCHW `[batch, in_channels, in_height, in_width]` +Batch Norm has been implemented as discussed in the paper: +https://arxiv.org/pdf/1502.03167.pdf +Can be used as a normalizer function for conv2d and fully_connected operations. +The required data format for this layer is one of the following: +1. NHWC `[batch, in_height, in_width, in_channels]` +2. NCHW `[batch, in_channels, in_height, in_width]` )DOC"); } @@ -295,9 +300,9 @@ class BatchNormGradOp : public framework::OperatorWithKernel { ctx->SetOutputDim(framework::GradVarName("Bias"), {C}); } - framework::DataType IndicateDataType( + protected: + framework::OpKernelType GetKernelType( const framework::ExecutionContext &ctx) const override { - VLOG(3) << "IndicateDataType " << this->Type(); const auto *var = ctx.InputVar(framework::GradVarName("Y")); if (var == nullptr) { PADDLE_THROW("can't find Y@GRAD"); @@ -311,7 +316,8 @@ class BatchNormGradOp : public framework::OperatorWithKernel { if (t == nullptr) { PADDLE_THROW("can't find Y@GRAD"); } - return framework::ToDataType(t->type()); + return framework::OpKernelType(framework::ToDataType(t->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/beam_search_decode_op.cc b/paddle/operators/beam_search_decode_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..1ba4dfcdaba498bfef98258f03664afebe14ec18 --- /dev/null +++ b/paddle/operators/beam_search_decode_op.cc @@ -0,0 +1,110 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/beam_search_decode_op.h" + +namespace paddle { +namespace operators { + +class BeamSearchDecodeOp : public framework::OperatorBase { + public: + BeamSearchDecodeOp(const std::string& type, + const framework::VariableNameMap& inputs, + const framework::VariableNameMap& outputs, + const framework::AttributeMap& attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const override { + framework::ExecutionContext ctx(*this, scope, dev_ctx); + const LoDTensorArray* ids = ctx.Input("Ids"); + const LoDTensorArray* scores = ctx.Input("Scores"); + const size_t step_num = ids->size(); + PADDLE_ENFORCE_GT(step_num, 0UL, + "beam search steps should be larger than 0"); + const size_t source_num = ids->at(0).lod().at(0).size() - 1; + PADDLE_ENFORCE_GT(source_num, 0UL, "source num should be larger than 0"); + + for (size_t i = 0; i < step_num; ++i) { + PADDLE_ENFORCE_EQ(ids->at(i).lod().size(), 2UL, + "Level of LodTensor should be 2"); + } + + // prepare output + LoDTensor* sentenceIds = ctx.Output("SentenceIds"); + LoDTensor* sentenceScores = ctx.Output("SentenceScores"); + + BeamSearchDecoder beam_search_decoder; + beam_search_decoder.PackAllSteps(*ids, *scores, sentenceIds, + sentenceScores); + } +}; + +class BeamSearchDecodeOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + BeamSearchDecodeOpProtoMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Ids", + "(LodTensorArray)" + "score of the candidate words in each step"); + AddInput("Scores", + "(LodTensorArray)" + "score of the candidate words in each step"); + AddOutput("SentenceIds", + "(LodTensor)" + "All possible result sentences of word ids"); + AddOutput("SentenceScores", + "(LodTensor)" + "All possible result sentences of word scores"); + AddComment(R"DOC( +Pack the result of Beam search op into SentenceIds and SentenceScores. +)DOC"); + } +}; + +class BeamSearchDecodeInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext* context) const override { + PADDLE_ENFORCE(context->HasInput("Ids"), + "BeamSearchDecodeOp must has input Ids"); + PADDLE_ENFORCE(context->HasInput("Scores"), + "BeamSearchDecodeOp must has input Scores"); + PADDLE_ENFORCE(context->HasOutput("SentenceIds"), + "BeamSearchDecodeOp must has output SentenceIds"); + PADDLE_ENFORCE(context->HasOutput("SentenceScores"), + "BeamSearchDecodeOp must has output SentenceScores"); + } +}; + +class BeamSearchDecodeInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind& op_desc, + framework::BlockDescBind* block) const override { + for (auto& o : op_desc.Output("SentenceIds")) { + block->Var(o)->SetType(framework::VarDesc::LOD_TENSOR); + } + for (auto& o : op_desc.Output("SentenceScores")) { + block->Var(o)->SetType(framework::VarDesc::LOD_TENSOR); + } + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(beam_search_decode, paddle::operators::BeamSearchDecodeOp, + paddle::operators::BeamSearchDecodeOpProtoMaker, + paddle::operators::BeamSearchDecodeInferShape, + paddle::operators::BeamSearchDecodeInferVarType, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/beam_search_decode_op.h b/paddle/operators/beam_search_decode_op.h new file mode 100644 index 0000000000000000000000000000000000000000..0f007ec22f9a66572971516a711317f348e1ec5a --- /dev/null +++ b/paddle/operators/beam_search_decode_op.h @@ -0,0 +1,280 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using LoDTensor = framework::LoDTensor; +using LoDTensorArray = framework::LoDTensorArray; + +// all the lod have 2 levels. +// The First is source level, the second is sentence level. +// source level describe how many candidate words for this source. +// sentence level describe these candidates belong to which prefix +const size_t kSourceLevel = 0; +const size_t kSentenceLevel = 1; + +template +struct BeamNode { + BeamNode(int64_t word_id, T score) : word_id_(word_id), score_(score) {} + + ~BeamNode() { + if (parent_) { + parent_->DropKid(this); + if (parent_->kids_.size() == 0UL) { + delete parent_; + } + } + VLOG(3) << "Delete BeamNode root with word_id:" << this->word_id_; + } + + void AppendTo(BeamNode* parent) { + parent_ = parent; + parent->kids_.insert(this); + } + + void DropKid(BeamNode* kid) { kids_.erase(kid); } + + BeamNode* parent_ = nullptr; + std::unordered_set kids_; + int64_t word_id_; + T score_; +}; + +template +using BeamNodeVector = std::vector>>; + +template +struct Sentence { + std::vector word_ids; + std::vector scores; +}; + +template +using SentenceVector = std::vector>; + +template +struct BeamSearchDecoder { + /** + * make a BeamNode and all it's related prefix BeanNode into a Sentence. + */ + Sentence MakeSentence(const BeamNode* node) const; + + /** + * Param: + * cur_ids: LoDTensor of One step for word ID + * cur_scores: LoDTensor of One Step for word score + * prefixes_list: prefixes for each source sentence. + * sentence_vector_list: result sentence_vector for each source sentence. + * Return: + * a new prefixes list for each source of current step + */ + std::vector> PackTwoSteps( + const LoDTensor& cur_ids, const LoDTensor& cur_scores, + std::vector>& prefixes_list, + std::vector>* sentence_vector_list) const; + + /** + * convert the result sentence_vector for each source sentence into two + * LodTensor. + * One is all candidate sentences with word id, one is all candidate sentences + * with word score. + * Param: + * sentence_vector_list: sentence_vector for each source sentence. + * id_tensor: result LoDTensor for sentences of id. + * score_tensor: result LoDTensor for sentences of score. + */ + void ConvertSentenceVectorToLodTensor( + std::vector> sentence_vector_list, LoDTensor* id_tensor, + LoDTensor* score_tensor) const; + + /** + * Pack all steps of id/score LodTensor into sentence LoDTensor + * it's main logic is: + * ```python + * prefix + * result_sentence + * result_lod_tensor + * + * for (step in steps): + * prefix = PackTwoSteps(prefix, step, &result_sentence) + * ConvertSentenceVectorToLodTensor(result_sentence, &result_lod_tensor) + * ``` + */ + void PackAllSteps(const LoDTensorArray& step_ids, + const LoDTensorArray& step_scores, LoDTensor* id_tensor, + LoDTensor* score_tensor) const; +}; + +template +Sentence BeamSearchDecoder::MakeSentence(const BeamNode* node) const { + Sentence sentence; + while (node != nullptr) { + sentence.word_ids.emplace_back(node->word_id_); + sentence.scores.emplace_back(node->score_); + node = node->parent_; + } + + std::reverse(std::begin(sentence.word_ids), std::end(sentence.word_ids)); + std::reverse(std::begin(sentence.scores), std::end(sentence.scores)); + + return sentence; +} + +template +std::vector> BeamSearchDecoder::PackTwoSteps( + const LoDTensor& cur_ids, const LoDTensor& cur_scores, + std::vector>& prefixes_list, + std::vector>* sentence_vector_list) const { + std::vector> result; + + for (size_t src_idx = 0; src_idx < cur_ids.lod()[kSourceLevel].size() - 1; + ++src_idx) { + size_t src_start = cur_ids.lod().at(kSourceLevel)[src_idx]; + size_t src_end = cur_ids.lod().at(kSourceLevel)[src_idx + 1]; + + BeamNodeVector beam_nodes; + + // if prefixes size is 0, it means this is the first step. In this step, + // all candidate id is the start of candidate sentences. + if (prefixes_list.empty()) { + PADDLE_ENFORCE_EQ(cur_ids.lod().at(kSourceLevel).back(), + cur_ids.lod().at(kSentenceLevel).back(), + "in the first step"); + for (size_t id_idx = src_start; id_idx < src_end; ++id_idx) { + beam_nodes.push_back(std::unique_ptr>(new BeamNode( + cur_ids.data()[id_idx], cur_scores.data()[id_idx]))); + } + } else { + BeamNodeVector& prefixes = prefixes_list[src_idx]; + SentenceVector& sentence_vector = (*sentence_vector_list)[src_idx]; + + PADDLE_ENFORCE_EQ(src_end - src_start, prefixes.size(), + "prefix and candidate set number should be the same"); + + auto candidate_offset = cur_ids.lod()[kSentenceLevel]; + for (size_t prefix_idx = 0; prefix_idx < prefixes.size(); ++prefix_idx) { + std::unique_ptr>& prefix = prefixes[prefix_idx]; + size_t candidate_start = candidate_offset[src_start + prefix_idx]; + size_t candidate_end = candidate_offset[src_start + prefix_idx + 1]; + if (candidate_start == candidate_end) { + VLOG(3) << "this sentence has no more candidate, " + "add to result sentence and rm it from beam tree"; + sentence_vector.push_back(MakeSentence(prefix.get())); + prefix.reset(); + } else { + for (size_t candidate_idx = candidate_start; + candidate_idx < candidate_end; ++candidate_idx) { + auto* candidate = + new BeamNode(cur_ids.data()[candidate_idx], + cur_scores.data()[candidate_idx]); + candidate->AppendTo(prefix.get()); + beam_nodes.push_back(std::unique_ptr>(candidate)); + } + prefix.release(); + } + } + } + result.push_back(std::move(beam_nodes)); + } + return result; +} + +template +void BeamSearchDecoder::ConvertSentenceVectorToLodTensor( + std::vector> sentence_vector_list, LoDTensor* id_tensor, + LoDTensor* score_tensor) const { + size_t src_num = sentence_vector_list.size(); + + PADDLE_ENFORCE_NE(src_num, 0, "src_num should not be 0"); + + std::vector source_level_lod = {0}; + std::vector sentence_level_lod = {0}; + std::vector id_data; + std::vector score_data; + + for (size_t src_idx = 0; src_idx < src_num; ++src_idx) { + for (Sentence& sentence : sentence_vector_list[src_idx]) { + id_data.insert(id_data.end(), sentence.word_ids.begin(), + sentence.word_ids.end()); + score_data.insert(score_data.end(), sentence.scores.begin(), + sentence.scores.end()); + sentence_level_lod.push_back(sentence_level_lod.back() + + sentence.word_ids.size()); + } + source_level_lod.push_back(source_level_lod.back() + + sentence_vector_list[src_idx].size()); + } + + auto cpu_place = new paddle::platform::CPUPlace(); + paddle::platform::CPUDeviceContext cpu_ctx(*cpu_place); + + framework::LoD lod; + lod.push_back(source_level_lod); + lod.push_back(sentence_level_lod); + + id_tensor->set_lod(lod); + id_tensor->Resize({static_cast(id_data.size())}); + id_tensor->mutable_data(paddle::platform::CPUPlace()); + id_tensor->CopyFromVector(id_data, cpu_ctx); + + score_tensor->set_lod(lod); + score_tensor->Resize({static_cast(score_data.size())}); + score_tensor->mutable_data(paddle::platform::CPUPlace()); + score_tensor->CopyFromVector(score_data, cpu_ctx); +} + +template +void BeamSearchDecoder::PackAllSteps(const LoDTensorArray& step_ids, + const LoDTensorArray& step_scores, + LoDTensor* id_tensor, + LoDTensor* score_tensor) const { + PADDLE_ENFORCE(!step_ids.empty(), "step num should be larger than 0"); + PADDLE_ENFORCE_EQ(step_ids.size(), step_scores.size(), + "step_ids and step_scores should be the same"); + const size_t step_num = step_ids.size(); + const size_t src_num = step_ids.at(0).lod().at(kSourceLevel).size() - 1; + + PADDLE_ENFORCE_GT(src_num, 0UL, "source num should be larger than 0"); + + // previous prefixes for each step, + // the init length is 0, means this is the first step. + std::vector> beamnode_vector_list(0); + std::vector> sentence_vector_list(src_num); + + // pack all steps for one batch first, then another batch + for (size_t step_id = 0; step_id < step_num; ++step_id) { + beamnode_vector_list = + PackTwoSteps(step_ids.at(step_id), step_scores.at(step_id), + beamnode_vector_list, &sentence_vector_list); + } + // append last beam_node to result + for (size_t src_idx = 0; src_idx < src_num; ++src_idx) { + for (auto& beam_node : beamnode_vector_list.at(src_idx)) { + sentence_vector_list[src_idx].push_back(MakeSentence(beam_node.get())); + beam_node.reset(); + } + } + + ConvertSentenceVectorToLodTensor(sentence_vector_list, id_tensor, + score_tensor); +} + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/beam_search_decode_op_test.cc b/paddle/operators/beam_search_decode_op_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..5ac23991f3c7768abaf94f3a4b750697de0ef114 --- /dev/null +++ b/paddle/operators/beam_search_decode_op_test.cc @@ -0,0 +1,221 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/beam_search_decode_op.h" +#include "gtest/gtest.h" + +using CPUPlace = paddle::platform::CPUPlace; +using LoD = paddle::framework::LoD; +using LoDTensor = paddle::framework::LoDTensor; +using LoDTensorArray = paddle::framework::LoDTensorArray; + +template +using BeamNode = paddle::operators::BeamNode; +template +using BeamSearchDecoder = paddle::operators::BeamSearchDecoder; +template +using Sentence = paddle::operators::Sentence; +template +using BeamNodeVector = paddle::operators::BeamNodeVector; +template +using SentenceVector = paddle::operators::SentenceVector; + +namespace paddle { +namespace test { + +void GenerateExample(const std::vector& level_0, + const std::vector& level_1, + const std::vector& data, LoDTensorArray* ids, + LoDTensorArray* scores) { + PADDLE_ENFORCE_EQ(level_0.back(), level_1.size() - 1, + "source level is used to describe candidate set"); + PADDLE_ENFORCE_EQ(level_1.back(), data.size(), + "the lowest level is used to describe data" + ", so it's last element should be data length"); + + CPUPlace place; + + LoD lod; + lod.push_back(level_0); + lod.push_back(level_1); + + // Ids + LoDTensor tensor_id; + tensor_id.set_lod(lod); + tensor_id.Resize({static_cast(data.size())}); + // malloc memory + int64_t* id_ptr = tensor_id.mutable_data(place); + for (size_t i = 0; i < data.size(); ++i) { + id_ptr[i] = static_cast(data.at(i)); + } + + // Scores + LoDTensor tensor_score; + tensor_score.set_lod(lod); + tensor_score.Resize({static_cast(data.size())}); + // malloc memory + float* score_ptr = tensor_score.mutable_data(place); + for (size_t i = 0; i < data.size(); ++i) { + score_ptr[i] = static_cast(data.at(i)); + } + + ids->push_back(tensor_id); + scores->push_back(tensor_score); +} + +} // namespace test +} // namespace paddle + +TEST(BeamSearchDecodeOp, DeleteBeamNode) { + auto* root = new BeamNode(0, 0); + auto* b1 = new BeamNode(1, 1); + auto* b2 = new BeamNode(2, 2); + auto* b3 = new BeamNode(3, 3); + + b1->AppendTo(root); + b2->AppendTo(root); + b3->AppendTo(b1); + + delete b3; + delete b2; +} + +TEST(BeamSearchDecodeOp, MakeSentence) { + auto* root = new BeamNode(0, 0); + auto* b1 = new BeamNode(1, 1); + auto* end = new BeamNode(2, 2); + b1->AppendTo(root); + end->AppendTo(b1); + + BeamSearchDecoder helper; + Sentence sentence = helper.MakeSentence(end); + delete end; + + std::vector expect_ids = {0, 1, 2}; + ASSERT_EQ(sentence.word_ids, expect_ids); + + std::vector expect_scores = {0, 1, 2}; + ASSERT_EQ(sentence.scores, expect_scores); +} + +TEST(BeamSearchDecodeOp, PackTwoStepsFistStep) { + CPUPlace place; + + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample( + std::vector{0, 2, 6}, std::vector{0, 1, 2, 3, 4, 5, 6}, + std::vector{1, 2, 3, 4, 5, 6}, &ids, &scores); + + std::vector> beamnode_vector_list; + std::vector> sentence_vector_list( + 2, SentenceVector()); + + BeamSearchDecoder helper; + beamnode_vector_list = helper.PackTwoSteps( + ids[0], scores[0], beamnode_vector_list, &sentence_vector_list); + ASSERT_EQ(beamnode_vector_list.size(), 2UL); + ASSERT_EQ(beamnode_vector_list[0].size(), 2UL); + ASSERT_EQ(beamnode_vector_list[1].size(), 4UL); +} + +TEST(BeamSearchDecodeOp, PackTwoSteps) { + CPUPlace place; + + // first source has three prefix + BeamNodeVector source0_prefixes; + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(1, 1))); + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(0, 0))); + source0_prefixes.push_back( + std::unique_ptr>(new BeamNode(3, 3))); + + // second source has two prefix + BeamNodeVector source1_prefixes; + source1_prefixes.push_back( + std::unique_ptr>(new BeamNode(4, 4))); + source1_prefixes.push_back( + std::unique_ptr>(new BeamNode(5, 5))); + + std::vector> beamnode_vector_list; + std::vector> sentence_vector_list( + 2, SentenceVector()); + + beamnode_vector_list.push_back(std::move(source0_prefixes)); + beamnode_vector_list.push_back(std::move(source1_prefixes)); + + // generate data for one step + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample(std::vector{0, 3, 5}, + std::vector{0, 1, 1, 3, 4, 5}, + std::vector{0, 1, 2, 3, 4}, &ids, &scores); + + BeamSearchDecoder helper1; + beamnode_vector_list = helper1.PackTwoSteps( + ids[0], scores[0], beamnode_vector_list, &sentence_vector_list); + + ASSERT_EQ(sentence_vector_list[0].size(), 1UL); + ASSERT_EQ(sentence_vector_list[1].size(), 0UL); + ASSERT_EQ(beamnode_vector_list[0].size(), 3UL); + ASSERT_EQ(beamnode_vector_list[1].size(), 2UL); +} + +TEST(BeamSearchDecodeOp, PackAllSteps) { + CPUPlace place; + + // we will constuct a sample data with 3 steps and 2 source sentences + LoDTensorArray ids; + LoDTensorArray scores; + + paddle::test::GenerateExample( + std::vector{0, 3, 6}, std::vector{0, 1, 2, 3, 4, 5, 6}, + std::vector{1, 2, 3, 4, 5, 6}, &ids, &scores); + paddle::test::GenerateExample( + std::vector{0, 3, 6}, std::vector{0, 1, 1, 3, 5, 5, 6}, + std::vector{0, 1, 2, 3, 4, 5}, &ids, &scores); + paddle::test::GenerateExample(std::vector{0, 3, 6}, + std::vector{0, 0, 1, 2, 3, 4, 5}, + std::vector{0, 1, 2, 3, 4}, &ids, &scores); + + ASSERT_EQ(ids.size(), 3UL); + ASSERT_EQ(scores.size(), 3UL); + + BeamSearchDecoder helper; + + LoDTensor id_tensor; + LoDTensor score_tensor; + helper.PackAllSteps(ids, scores, &id_tensor, &score_tensor); + + LoD lod = id_tensor.lod(); + std::vector expect_source_lod = {0, 4, 8}; + EXPECT_EQ(lod[0], expect_source_lod); + std::vector expect_sentence_lod = {0, 1, 3, 6, 9, 10, 13, 16, 19}; + EXPECT_EQ(lod[1], expect_sentence_lod); + // 2| 1, 0| 3, 1, 0| 3, 2, 1| 5| 4, 3, 2| 4, 4, 3| 6, 5, 4 + std::vector expect_data = {2, 1, 0, 3, 1, 0, 3, 2, 1, 5, + 4, 3, 2, 4, 4, 3, 6, 5, 4}; + ASSERT_EQ(id_tensor.dims()[0], static_cast(expect_data.size())); + for (size_t i = 0; i < expect_data.size(); ++i) { + ASSERT_EQ(id_tensor.data()[i], + static_cast(expect_data[i])); + } + for (int64_t i = 0; i < id_tensor.dims()[0]; ++i) { + ASSERT_EQ(score_tensor.data()[i], + static_cast(id_tensor.data()[i])); + } +} diff --git a/paddle/operators/bilinear_tensor_product_op.cc b/paddle/operators/bilinear_tensor_product_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..c65ba7eb262f3aabe2c00837b79806c0b40b60fd --- /dev/null +++ b/paddle/operators/bilinear_tensor_product_op.cc @@ -0,0 +1,159 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/bilinear_tensor_product_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class BilinearTensorProductOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null."); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto weight_dims = ctx->GetInputDim("Weight"); + + PADDLE_ENFORCE_EQ(x_dims.size(), 2UL, "The input(X) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ(y_dims.size(), 2UL, "The input(Y) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ(weight_dims.size(), 3UL, + "The input(Weight) must be a 3D tensor."); + PADDLE_ENFORCE_EQ(x_dims[0], y_dims[0], + "The first dimension(batch_size) of input(X) must be " + "equal to the first dimension of the input(Y)."); + PADDLE_ENFORCE_EQ(x_dims[1], weight_dims[1], + "The second dimension of input(X) must be equal to " + "the second dimension of the input(Weight)."); + PADDLE_ENFORCE_EQ(y_dims[1], weight_dims[2], + "The second dimension of input(Y) must be equal to " + "the third dimension of the input(Weight)."); + + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + PADDLE_ENFORCE(bias_dims.size() == 2UL && bias_dims[0] == 1UL, + "The Input(Bias) must be a 2-D tensor with " + "the 2nd dimension fixed to 1 (a row vector)."); + PADDLE_ENFORCE_EQ(bias_dims[1], weight_dims[0], + "The second dimension of input(Bias) must be equal " + "to the first dimension of the input(Weight)."); + } + + ctx->SetOutputDim("Out", {x_dims[0], weight_dims[0]}); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class BilinearTensorProductOpMaker : public framework::OpProtoAndCheckerMaker { + public: + BilinearTensorProductOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The first input of bilinear_tensor_product operator."); + AddInput("Y", "The second input of bilinear_tensor_product operator."); + AddInput("Weight", + "The learnable parameters of bilinear_tensor_product operator."); + AddInput("Bias", "The learnable bias of bilinear_tensor_product operator.") + .AsDispensable(); + AddOutput("Out", "The output of bilinear_tensor_product operator."); + AddComment(R"DOC( +Bilinear Tensor Product operator. +Given input X and Y, a 3D tensor weight, and bias. Each column of the +output is computed by one slice i = 1, . . . , k of the tensor: + + M = (X W_i) \cdot Y + Out_i = \sum_i {M_i} + Bias_i + +)DOC"); + } +}; + +class BilinearTensorProductOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) should not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null."); + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto weight_dims = ctx->GetInputDim("Weight"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + + PADDLE_ENFORCE_EQ(out_dims.size(), 2UL, + "The input(Out@GRAD) must be a 2D Tensor."); + PADDLE_ENFORCE_EQ( + x_dims[0], out_dims[0], + "The first dimension(batch_size) of input(Out@GRAD) must be " + "equal to the first dimension of the Input(X)."); + PADDLE_ENFORCE_EQ( + weight_dims[0], out_dims[1], + "The second dimension of input(Out@GRAD) must be equal to " + "the third dimension of the Input(Weight)."); + + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + PADDLE_ENFORCE_EQ( + bias_dims[1], out_dims[1], + "The second dimension of input(Out@GRAD) must be equal to " + "the second dimension of the Input(Bias)."); + auto bias_grad_name = framework::GradVarName("Bias"); + if (ctx->HasOutput(bias_grad_name)) + ctx->SetOutputDim(bias_grad_name, bias_dims); + } + + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + auto weight_grad_name = framework::GradVarName("Weight"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); + } + if (ctx->HasOutput(weight_grad_name)) { + ctx->SetOutputDim(weight_grad_name, weight_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(bilinear_tensor_product, ops::BilinearTensorProductOp, + ops::BilinearTensorProductOpMaker, bilinear_tensor_product_grad, + ops::BilinearTensorProductOpGrad); +REGISTER_OP_CPU_KERNEL( + bilinear_tensor_product, + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); +REGISTER_OP_CPU_KERNEL( + bilinear_tensor_product_grad, + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/math/detail/hl_avx_functions.h b/paddle/operators/bilinear_tensor_product_op.cu similarity index 52% rename from paddle/operators/math/detail/hl_avx_functions.h rename to paddle/operators/bilinear_tensor_product_op.cu index 35f4eabb4c07c6cc9d2edded02e5b6290b1232f8..858d2668d01379afe8082cd1eda32a2a5d09bd18 100644 --- a/paddle/operators/math/detail/hl_avx_functions.h +++ b/paddle/operators/bilinear_tensor_product_op.cu @@ -12,21 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef HL_AVX_FUNCTIONS_H_ -#define HL_AVX_FUNCTIONS_H_ - -#include - -namespace hppl { -__m256 relu(const __m256 a); -__m256 sigmoid(const __m256 a); -__m256 tanh(const __m256 a); -__m256 linear(const __m256 a); - -__m256 relu(const __m256 a, const __m256 b); -__m256 sigmoid(const __m256 a, const __m256 b); -__m256 tanh(const __m256 a, const __m256 b); -__m256 linear(const __m256 a, const __m256 b); -} // namespace hppl - -#endif // HL_AVX_FUNCTIONS_H_ +#define EIGEN_USE_GPU +#include "paddle/operators/bilinear_tensor_product_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + bilinear_tensor_product, + ops::BilinearTensorProductKernel, + ops::BilinearTensorProductKernel); +REGISTER_OP_GPU_KERNEL( + bilinear_tensor_product_grad, + ops::BilinearTensorProductGradKernel, + ops::BilinearTensorProductGradKernel); diff --git a/paddle/operators/bilinear_tensor_product_op.h b/paddle/operators/bilinear_tensor_product_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ffa4f43a327418498c1f110504127e7d2878409d --- /dev/null +++ b/paddle/operators/bilinear_tensor_product_op.h @@ -0,0 +1,184 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +template +using EigenMatrix = framework::EigenMatrix; + +template +class BilinearTensorProductKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* weight = ctx.Input("Weight"); + auto* bias = ctx.Input("Bias"); + auto* out = ctx.Output("Out"); + out->mutable_data(ctx.GetPlace()); + + auto y_mat = EigenMatrix::From(*y); + auto output_mat = EigenMatrix::From(*out); + + auto batch_size = x->dims()[0]; + auto weight_dims = weight->dims(); + int out_dim = weight_dims[0]; + auto x_dim = weight_dims[1]; + auto y_dim = weight_dims[2]; + auto place = ctx.GetEigenDevice(); + + // Create the intermediate variable to caculate the result of + // Input(X) multiplied by Input(Weight_i), the formula is: + // left_mul = X Weight_i. + Tensor left_mul; + left_mul.mutable_data(framework::make_ddim({batch_size, y_dim}), + ctx.GetPlace()); + auto left_mul_mat = EigenMatrix::From(left_mul); + + for (int i = 0; i < out_dim; ++i) { + auto output_col_vec = output_mat.chip(i, 1); + Tensor weight_mat = + weight->Slice(i, i + 1).Resize(framework::make_ddim({x_dim, y_dim})); + math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, + batch_size, y_dim, x_dim, 1, x->data(), + weight_mat.data(), 0, left_mul.data()); + output_col_vec.device(place) = + (left_mul_mat * y_mat).sum(Eigen::DSizes(1)); + } + if (bias) { + auto bias_vec = EigenMatrix::From(*bias); + Eigen::DSizes bcast(batch_size, 1); + output_mat.device(place) = bias_vec.broadcast(bcast) + output_mat; + } + } +}; + +template +class BilinearTensorProductGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + const Tensor* x = ctx.Input("X"); + const Tensor* y = ctx.Input("Y"); + const Tensor* weight = ctx.Input("Weight"); + Tensor* d_x = ctx.Output(framework::GradVarName("X")); + Tensor* d_y = ctx.Output(framework::GradVarName("Y")); + Tensor* d_weight = ctx.Output(framework::GradVarName("Weight")); + Tensor* d_bias = ctx.Output(framework::GradVarName("Bias")); + const Tensor* d_out = ctx.Input(framework::GradVarName("Out")); + + auto batch_size = x->dims()[0]; + auto weight_dims = weight->dims(); + int out_dim = weight_dims[0]; + auto x_dim = weight_dims[1]; + auto y_dim = weight_dims[2]; + + auto x_mat = EigenMatrix::From(*x); + auto y_mat = EigenMatrix::From(*y); + auto d_out_mat = EigenMatrix::From(*d_out); + auto place = ctx.GetEigenDevice(); + + // Create the intermediate variable to caculate the Output(Y@Grad). + Tensor x_scale; + x_scale.mutable_data(framework::make_ddim({batch_size, x_dim}), + ctx.GetPlace()); + auto x_scale_mat = EigenMatrix::From(x_scale); + + // Create the intermediate variable to caculate the Output(X@Grad). + Tensor y_scale; + y_scale.mutable_data(framework::make_ddim({batch_size, y_dim}), + ctx.GetPlace()); + auto y_scale_mat = EigenMatrix::From(y_scale); + + math::SetConstant set_zero; + + // Set Output(X@Grad) be zero. + if (d_x) { + d_x->mutable_data(ctx.GetPlace()); + set_zero(ctx.device_context(), d_x, static_cast(0)); + } + + // Set Output(Y@Grad) be zero. + if (d_y) { + d_y->mutable_data(ctx.GetPlace()); + set_zero(ctx.device_context(), d_y, static_cast(0)); + } + + // Caculate the Output(X@Grad) and Output(Y@Grad). + if (d_x || d_y) { + Eigen::DSizes bcast_for_x(1, y_dim); + Eigen::DSizes bcast_for_y(1, x_dim); + for (int i = 0; i < out_dim; ++i) { + Tensor weight_i = weight->Slice(i, i + 1).Resize( + framework::make_ddim({x_dim, y_dim})); + auto output_vec = d_out_mat.chip(i, 1); + if (d_x) { + y_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_x) * + y_mat; + math::gemm(ctx.device_context(), CblasNoTrans, CblasTrans, + batch_size, x_dim, y_dim, 1, y_scale.data(), + weight_i.data(), 1, d_x->data()); + } + if (d_y) { + x_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_y) * + x_mat; + math::gemm(ctx.device_context(), CblasNoTrans, CblasNoTrans, + batch_size, y_dim, x_dim, 1, x_scale.data(), + weight_i.data(), 1, d_y->data()); + } + } + } + + // Caculate the gradient of Input(Weight). + if (d_weight) { + d_weight->mutable_data(ctx.GetPlace()); + Eigen::DSizes bcast_for_weight(1, x_dim); + for (int i = 0; i < out_dim; ++i) { + Tensor d_weight_i = d_weight->Slice(i, i + 1).Resize( + framework::make_ddim({x_dim, y_dim})); + auto output_vec = d_out_mat.chip(i, 1); + x_scale_mat.device(place) = + output_vec.reshape(Eigen::DSizes(batch_size, 1)) + .broadcast(bcast_for_weight) * + x_mat; + math::gemm(ctx.device_context(), CblasTrans, CblasNoTrans, + x_dim, y_dim, batch_size, 1, x_scale.data(), + y->data(), 0, d_weight_i.data()); + } + } + + // Caculate the gradient of Input(Bias). + if (d_bias) { + d_bias->mutable_data(ctx.GetPlace()); + auto d_bias_mat = EigenMatrix::From(*d_bias); + d_bias_mat.device(place) = d_out_mat.sum(Eigen::DSizes(0)); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/cast_op.cc b/paddle/operators/cast_op.cc index 19187894c3f4803ef241d5e0c159852c0d9687da..70ee7861bab3a982eae60dd85b10c2e41f5827d0 100644 --- a/paddle/operators/cast_op.cc +++ b/paddle/operators/cast_op.cc @@ -23,13 +23,17 @@ class CastOpProtoMaker : public framework::OpProtoAndCheckerMaker { CastOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensor of cast op"); - AddOutput("Out", "the output tensor of cast op"); - AddComment(R"DOC(Cast operator. -cast the input tensor to other data type. -)DOC"); + AddInput("X", "The input tensor of cast op"); + AddOutput("Out", "The output tensor of cast op"); AddAttr("out_data_type", "output data type"); AddAttr("in_data_type", "input data type"); + AddComment(R"DOC( +Cast Operator. + +This Operator casts the input tensor to another data type and +returns tha Output Tensor. + +)DOC"); } }; diff --git a/paddle/operators/chunk_eval_op.cc b/paddle/operators/chunk_eval_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..309660b01fe7052de2f9300acdf00779d0228221 --- /dev/null +++ b/paddle/operators/chunk_eval_op.cc @@ -0,0 +1,145 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/chunk_eval_op.h" + +namespace paddle { +namespace operators { + +class ChunkEvalOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Inference"), + "Input(Inference) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Label"), + "Input(Label) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Precision"), + "Output(Precision) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Recall"), + "Output(Recall) of ChunkEvalOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("F1-Score"), + "Output(F1-Score) of ChunkEvalOp should not be null."); + + auto inference_dim = ctx->GetInputDim("Inference"); + auto label_dim = ctx->GetInputDim("Label"); + + PADDLE_ENFORCE(inference_dim == label_dim, + "Inference's shape must be the same as Label's shape."); + + ctx->SetOutputDim("Precision", {1}); + ctx->SetOutputDim("Recall", {1}); + ctx->SetOutputDim("F1-Score", {1}); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType(framework::DataType::FP32, + ctx.device_context()); + } +}; + +class ChunkEvalOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ChunkEvalOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Inference", + "(Tensor, default: Tensor). Predictions from the network."); + AddInput("Label", + "(Tensor, default: Tensor). The true tag sequences."); + AddOutput("Precision", + "(float). The evaluated precision (called positive predictive " + "value) of chunks on the given mini-batch."); + AddOutput("Recall", + "(float). The evaluated recall (true positive rate or " + "sensitivity) of chunks on the given mini-batch."); + AddOutput("F1-Score", + "(float). The evaluated F1-Score on the given mini-batch."); + AddAttr("num_chunk_types", + "(int). The number of chunk type. See below for details."); + AddAttr( + "chunk_scheme", + "(string, default IOB). The labeling scheme indicating " + "how to encode the chunks. Must be IOB, IOE, IOBES or plain. See below " + "for details.") + .SetDefault("IOB"); + AddAttr>("excluded_chunk_types", + "(list) A list including chunk type ids " + "indicating chunk types that are not counted. " + "See below for details.") + .SetDefault(std::vector{}); + AddComment(R"DOC( +For some basics of chunking, please refer to +‘Chunking with Support Vector Mechines ’. + + +CheckEvalOp computes the precision, recall, and F1-score of chunk detection, +and supports IOB, IOE, IOBES and IO (also known as plain) tagging schemes. +Here is a NER example of labeling for these tagging schemes: + + Li Ming works at Agricultural Bank of China in Beijing. + IO: I-PER I-PER O O I-ORG I-ORG I-ORG I-ORG O I-LOC + IOB: B-PER I-PER O O B-ORG I-ORG I-ORG I-ORG O B-LOC + IOE: I-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O E-LOC + IOBES: B-PER E-PER O O I-ORG I-ORG I-ORG E-ORG O S-LOC + +There are three chunk types(named entity types) including PER(person), ORG(orgnazation) +and LOC(LOCATION), and we can see that the labels have the form -. + +Since the calculations actually use label ids rather than labels, extra attention +should be paid when mapping labels to ids to make CheckEvalOp work. The key point +is that the listed equations are satisfied by ids. + + tag_type = label % num_tag_type + chunk_type = label / num_tag_type + +where `num_tag_type` is the num of tag types in the tagging scheme, `num_chunk_type` +is the num of chunk types, and `tag_type` get its value from the following table. + + Scheme Begin Inside End Single + plain 0 - - - + IOB 0 1 - - + IOE - 0 1 - + IOBES 0 1 2 3 + +Still use NER as example, assuming the tagging scheme is IOB while chunk types are ORG, +PER and LOC. To satisfy the above equations, the label map can be like this: + + B-ORG 0 + I-ORG 1 + B-PER 2 + I-PER 3 + B-LOC 4 + I-LOC 5 + O 6 + +It’s not hard to verify the equations noting that the num of chunk types +is 3 and the num of tag types in IOB scheme is 2. For example, the label +id of I-LOC is 5, the tag type id of I-LOC is 1, and the chunk type id of +I-LOC is 2, which consistent with the results from the equations. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(chunk_eval, ops::ChunkEvalOp, + ops::ChunkEvalOpMaker); +REGISTER_OP_CPU_KERNEL(chunk_eval, + ops::ChunkEvalKernel); diff --git a/paddle/operators/chunk_eval_op.h b/paddle/operators/chunk_eval_op.h new file mode 100644 index 0000000000000000000000000000000000000000..81aa07817b673b2ff85a35a51cc43742b7ad7fed --- /dev/null +++ b/paddle/operators/chunk_eval_op.h @@ -0,0 +1,219 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; + +template +class ChunkEvalKernel : public framework::OpKernel { + public: + struct Segment { + int begin; + int end; + int type; + bool operator==(const Segment& y) const { + return begin == y.begin && end == y.end && type == y.type; + } + }; + + void GetSegments(const int* label, int length, std::vector& segments, + int num_chunk_types, int num_tag_types, int other_chunk_type, + int tag_begin, int tag_inside, int tag_end, + int tag_single) const { + segments.clear(); + segments.reserve(length); + int chunk_start = 0; + bool in_chunk = false; + int tag = -1; + int type = other_chunk_type; + for (int i = 0; i < length; ++i) { + int prev_tag = tag; + int prev_type = type; + PADDLE_ENFORCE_LE(label[i], num_chunk_types * num_tag_types); + tag = label[i] % num_tag_types; + type = label[i] / num_tag_types; + if (in_chunk && ChunkEnd(prev_tag, prev_type, tag, type, other_chunk_type, + tag_begin, tag_inside, tag_end, tag_single)) { + Segment segment{ + chunk_start, // begin + i - 1, // end + prev_type, + }; + segments.push_back(segment); + in_chunk = false; + } + if (ChunkBegin(prev_tag, prev_type, tag, type, other_chunk_type, + tag_begin, tag_inside, tag_end, tag_single)) { + chunk_start = i; + in_chunk = true; + } + } + if (in_chunk) { + Segment segment{ + chunk_start, // begin + length - 1, // end + type, + }; + segments.push_back(segment); + } + } + + bool ChunkEnd(int prev_tag, int prev_type, int tag, int type, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single) const { + if (prev_type == other_chunk_type) return false; + if (type == other_chunk_type) return true; + if (type != prev_type) return true; + if (prev_tag == tag_begin) return tag == tag_begin || tag == tag_single; + if (prev_tag == tag_inside) return tag == tag_begin || tag == tag_single; + if (prev_tag == tag_end) return true; + if (prev_tag == tag_single) return true; + return false; + } + + bool ChunkBegin(int prev_tag, int prev_type, int tag, int type, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single) const { + if (prev_type == other_chunk_type) return type != other_chunk_type; + if (type == other_chunk_type) return false; + if (type != prev_type) return true; + if (tag == tag_begin) return true; + if (tag == tag_inside) return prev_tag == tag_end || prev_tag == tag_single; + if (tag == tag_end) return prev_tag == tag_end || prev_tag == tag_single; + if (tag == tag_single) return true; + return false; + } + + void Compute(const framework::ExecutionContext& context) const override { + // initialize to parse configurations + int num_chunk_types, num_tag_types; + int other_chunk_type; + int tag_begin, tag_inside, tag_end, tag_single; + std::vector label_segments; + std::vector output_segments; + std::set excluded_chunk_types; + int64_t num_output_segments = 0; + int64_t num_label_segments = 0; + int64_t num_correct = 0; + if (context.Attr("chunk_scheme") == "IOB") { + num_tag_types = 2; + tag_begin = 0; + tag_inside = 1; + tag_end = -1; + tag_single = -1; + } else if (context.Attr("chunk_scheme") == "IOE") { + num_tag_types = 2; + tag_begin = -1; + tag_inside = 0; + tag_end = 1; + tag_single = -1; + } else if (context.Attr("chunk_scheme") == "IOBES") { + num_tag_types = 4; + tag_begin = 0; + tag_inside = 1; + tag_end = 2; + tag_single = 3; + } else if (context.Attr("chunk_scheme") == "plain") { + num_tag_types = 1; + tag_begin = -1; + tag_inside = -1; + tag_end = -1; + tag_single = -1; + } else { + PADDLE_THROW("Unknown chunk scheme."); + } + other_chunk_type = num_chunk_types = context.Attr("num_chunk_types"); + excluded_chunk_types.insert( + context.Attr>("excluded_chunk_types").begin(), + context.Attr>("excluded_chunk_types").end()); + + auto* inference = context.Input("Inference"); + auto* label = context.Input("Label"); + auto* precision = context.Output("Precision"); + auto* recall = context.Output("Recall"); + auto* f1 = context.Output("F1-Score"); + + const int* inference_data = inference->data(); + const int* label_data = label->data(); + T* precision_data = precision->mutable_data(context.GetPlace()); + T* racall_data = recall->mutable_data(context.GetPlace()); + T* f1_data = f1->mutable_data(context.GetPlace()); + + auto lod = label->lod(); + PADDLE_ENFORCE_EQ(lod.size(), 1UL, "Only support one level sequence now."); + PADDLE_ENFORCE(lod == inference->lod(), + "LoD must be same between Inference and Label."); + int num_sequences = lod[0].size() - 1; + for (int i = 0; i < num_sequences; ++i) { + int seq_length = lod[0][i + 1] - lod[0][i]; + EvalOneSeq(inference_data + lod[0][i], label_data + lod[0][i], seq_length, + output_segments, label_segments, num_output_segments, + num_label_segments, num_correct, num_chunk_types, + num_tag_types, other_chunk_type, tag_begin, tag_inside, + tag_end, tag_single, excluded_chunk_types); + } + *precision_data = !num_output_segments ? 0 : static_cast(num_correct) / + num_output_segments; + *racall_data = !num_label_segments ? 0 : static_cast(num_correct) / + num_label_segments; + *f1_data = !num_correct ? 0 : 2 * (*precision_data) * (*racall_data) / + ((*precision_data) + (*racall_data)); + } + + void EvalOneSeq(const int* output, const int* label, int length, + std::vector& output_segments, + std::vector& label_segments, + int64_t& num_output_segments, int64_t& num_label_segments, + int64_t& num_correct, int num_chunk_types, int num_tag_types, + int other_chunk_type, int tag_begin, int tag_inside, + int tag_end, int tag_single, + const std::set& excluded_chunk_types) const { + GetSegments(output, length, output_segments, num_chunk_types, num_tag_types, + other_chunk_type, tag_begin, tag_inside, tag_end, tag_single); + GetSegments(label, length, label_segments, num_chunk_types, num_tag_types, + other_chunk_type, tag_begin, tag_inside, tag_end, tag_single); + size_t i = 0, j = 0; + while (i < output_segments.size() && j < label_segments.size()) { + if (output_segments[i] == label_segments[j] && + excluded_chunk_types.count(output_segments[i].type) != 1) { + ++num_correct; + } + if (output_segments[i].end < label_segments[j].end) { + ++i; + } else if (output_segments[i].end > label_segments[j].end) { + ++j; + } else { + ++i; + ++j; + } + } + for (auto& segment : label_segments) { + if (excluded_chunk_types.count(segment.type) != 1) ++num_label_segments; + } + for (auto& segment : output_segments) { + if (excluded_chunk_types.count(segment.type) != 1) ++num_output_segments; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/clip_by_norm_op.cc b/paddle/operators/clip_by_norm_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..d9fc532e39500fa397be80396b075e866bad9362 --- /dev/null +++ b/paddle/operators/clip_by_norm_op.cc @@ -0,0 +1,70 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_by_norm_op.h" + +namespace paddle { +namespace operators { + +class ClipByNormOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of ClipByNormOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of ClipByNormOp should not be null."); + auto max_norm = ctx->Attrs().Get("max_norm"); + PADDLE_ENFORCE_GT(max_norm, 0, "max_norm should be greater than 0."); + auto x_dims = ctx->GetInputDim("X"); + ctx->SetOutputDim("Out", x_dims); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class ClipByNormOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ClipByNormOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor) The input of clip_by_norm op." + "The number of dimensions must be between [1, 9]."); + AddOutput("Out", + "(Tensor) The output of clip_by_norm op with shape as input(X)"); + AddAttr("max_norm", "(float) The maximum norm value."); + AddComment(R"DOC( +ClipByNorm operator limits the L2 norm of the input 'X' within 'max_norm'. +If the L2 norm of 'X' is less than or equal to 'max_norm', 'Out' will be +the same as 'X'. If the L2 norm of 'X' is greater than 'max_norm', 'X' will +be linearly scaled to make the L2 norm of 'Out' equal to 'max_norm', as +shown in the following formula： + +'Out' = 'max_norm' * 'X' / norm('X'), + +where norm('X') represents the L2 norm of 'X'. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(clip_by_norm, ops::ClipByNormOp, + ops::ClipByNormOpMaker); +REGISTER_OP_CPU_KERNEL( + clip_by_norm, ops::ClipByNormKernel); diff --git a/paddle/operators/increment_op.cu b/paddle/operators/clip_by_norm_op.cu similarity index 80% rename from paddle/operators/increment_op.cu rename to paddle/operators/clip_by_norm_op.cu index 659c380d147a36650452bea23b30cbcf1ff516ee..2593a24ebbf56ecd286a726e527d2414247576e8 100644 --- a/paddle/operators/increment_op.cu +++ b/paddle/operators/clip_by_norm_op.cu @@ -12,8 +12,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/increment_op.h" +#include "paddle/operators/clip_by_norm_op.h" +namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL( - increment, - paddle::operators::IncrementKernel); + clip_by_norm, ops::ClipByNormKernel); diff --git a/paddle/operators/clip_by_norm_op.h b/paddle/operators/clip_by_norm_op.h new file mode 100644 index 0000000000000000000000000000000000000000..b26476cae9b5b2fa290bc9186b9a64c48ba703d6 --- /dev/null +++ b/paddle/operators/clip_by_norm_op.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +class ClipByNormKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max_norm = context.Attr("max_norm"); + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + output->mutable_data(context.GetPlace()); + + auto x = EigenVector::Flatten(*input); + auto out = EigenVector::Flatten(*output); + auto x_norm = x.square().sum().sqrt(); + auto place = context.GetEigenDevice(); + + auto temp = (x_norm <= max_norm).template cast().eval(); + auto scaling = temp + (static_cast(1) - temp) * max_norm / x_norm; + Eigen::array one_dim{{1}}; + Eigen::DSizes m_dsize(input->numel()); + out.device(place) = x * scaling.reshape(one_dim).broadcast(m_dsize); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc index f80204c6833d6436f2cf21610beea45b36787eea..3e9066ceb2a4a4dc19fdf5ef02bb7fadaab4bfff 100644 --- a/paddle/operators/clip_op.cc +++ b/paddle/operators/clip_op.cc @@ -49,8 +49,11 @@ class ClipOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr( "max", "(float)Maximum value, above which element is replaced by max"); AddComment(R"DOC( -Clip operator limits the given input within an interval. The interval is +Clip Operator. + +The clip operator limits the value of given input within an interval. The interval is specified with arguments 'min' and 'max'. + )DOC"); } }; diff --git a/paddle/operators/compare_op.cc b/paddle/operators/compare_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..716b5ee92d0d8737d2069460f53989f691ff7c77 --- /dev/null +++ b/paddle/operators/compare_op.cc @@ -0,0 +1,98 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/compare_op.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { +template +class CompareOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + CompareOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + OpComment comment; + AddInput("X", + string::Sprintf("(LoDTensor) the left hand operand of %s operator", + comment.type)); + AddInput("Y", string::Sprintf( + "(LoDTensor) the right hand operand of %s operator", + comment.type)); + AddOutput("Out", string::Sprintf( + "(LoDTensor) n-dim bool tensor. Each element is %s", + comment.equation)); + AddComment(string::Sprintf(R"DOC(%s Operator + +It operates element-wise on X and Y, and returns the Out. Each of them is a +N-dim tensor. X and Y could be any type. The each element of the Out tensor is +calculated by %s +)DOC", + comment.type, comment.equation)); + } +}; + +template +class CompareOpInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + OpComment comment; + PADDLE_ENFORCE(context->HasInput("X"), "%s operator must has input X", + comment.type); + PADDLE_ENFORCE(context->HasInput("Y"), "%s operator must has input Y", + comment.type); + auto dim_x = context->GetInputDim("X"); + auto dim_y = context->GetInputDim("Y"); + PADDLE_ENFORCE_EQ(framework::product(dim_x), framework::product(dim_y), + "The number of elements in X and Y should be same"); + + context->SetOutputDim("Out", context->GetInputDim("X")); + context->ShareLoD("X", "Out"); + } +}; + +class CompareOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + framework::OpKernelType kt = OperatorWithKernel::GetKernelType(ctx); + // CompareOp kernel's device type is decided by input tensor place + kt.place_ = ctx.Input("X")->place(); + return kt; + } +}; + +} // namespace operators +} // namespace paddle + +#define REGISTER_LOGICAL_OP(op_type, _equation) \ + struct _##op_type##Comment { \ + static char type[]; \ + static char equation[]; \ + }; \ + char _##op_type##Comment::type[]{#op_type}; \ + char _##op_type##Comment::equation[]{_equation}; \ + REGISTER_OPERATOR( \ + op_type, ::paddle::operators::CompareOp, \ + ::paddle::operators::CompareOpProtoMaker<_##op_type##Comment>, \ + ::paddle::operators::CompareOpInferShape<_##op_type##Comment>, \ + ::paddle::framework::EmptyGradOpMaker); + +REGISTER_LOGICAL_OP(less_than, "Out = X < Y"); +REGISTER_LOGICAL_KERNEL(less_than, CPU, paddle::operators::LessThanFunctor); +REGISTER_LOGICAL_OP(equal, "Out = X == Y"); +REGISTER_LOGICAL_KERNEL(equal, CPU, paddle::operators::EqualFunctor); diff --git a/paddle/operators/compare_op.cu b/paddle/operators/compare_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..42a5bb2f45fd389f60c3dc034cade7f56a907e35 --- /dev/null +++ b/paddle/operators/compare_op.cu @@ -0,0 +1,18 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/compare_op.h" + +REGISTER_LOGICAL_KERNEL(less_than, GPU, paddle::operators::LessThanFunctor); +REGISTER_LOGICAL_KERNEL(equal, GPU, paddle::operators::EqualFunctor); diff --git a/paddle/operators/compare_op.h b/paddle/operators/compare_op.h new file mode 100644 index 0000000000000000000000000000000000000000..04e04e347b398abb5fb66876bf801b1eee688ec6 --- /dev/null +++ b/paddle/operators/compare_op.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +template +struct LessThanFunctor { + using ELEM_TYPE = T; + HOSTDEVICE bool operator()(const T& a, const T& b) const { return a < b; } +}; + +template +struct EqualFunctor { + using ELEM_TYPE = T; + HOSTDEVICE bool operator()(const T& a, const T& b) const { + if (std::is_floating_point::value) { + // This branch will be optimized while compiling if T is integer. It is + // safe to cast a and b to double. + return fabs(static_cast(a - b)) < 1e-8; + } else { + return (a == b); + } + } +}; + +template +class CompareOpKernel + : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + using T = typename Functor::ELEM_TYPE; + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + auto* out = context.Output("Out"); + Functor binary_func; + platform::Transform trans; + trans(context.device_context(), x->data(), x->data() + x->numel(), + y->data(), out->mutable_data(context.GetPlace()), + binary_func); + } +}; + +} // namespace operators +} // namespace paddle + +#define REGISTER_LOGICAL_KERNEL(op_type, dev, functor) \ + REGISTER_OP_##dev##_KERNEL( \ + op_type, \ + ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ + functor>, \ + ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ + functor>, \ + ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ + functor>, \ + ::paddle::operators::CompareOpKernel<::paddle::platform::dev##Place, \ + functor>); diff --git a/paddle/operators/concat_op.cc b/paddle/operators/concat_op.cc index e11e51b4583817ef50cd447dbcf4c7202a152422..5f052689251bc023df635d41c1e64a660a0aa488 100644 --- a/paddle/operators/concat_op.cc +++ b/paddle/operators/concat_op.cc @@ -56,20 +56,24 @@ class ConcatOpMaker : public framework::OpProtoAndCheckerMaker { public: ConcatOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of concat operator.").AsDuplicable(); - AddOutput("Out", "the output tensor of concat operator."); - AddComment(R"DOC( - Join the input tensors along with the axis. - Examples: - Input[0] = [[1,2],[3,4]] - Input[1] = [[5,6]] - axis = 0 - Output = [[1,2], - [3,4], - [5,6]] - )DOC"); - AddAttr("axis", "The axis which the inputs will be joined with.") + AddInput("X", "Input tensors of concat operator.").AsDuplicable(); + AddOutput("Out", "Output tensor of concat operator."); + AddAttr("axis", + "The axis along which the input tensors will be concatenated.") .SetDefault(0); + AddComment(R"DOC( +Concat Operator. + +Concatenate the input tensors along dimension axis. +Examples: + Input[0] = [[1,2],[3,4]] + Input[1] = [[5,6]] + axis = 0 + Output = [[1,2], + [3,4], + [5,6]] + +)DOC"); } }; diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index adcd867f502d166f851926fde602dbb3fed9b48e..b809bdc3a0fea727f2fb6ea0a55672ee9b0bbd04 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -216,11 +216,12 @@ class CondOpProtoAndCheckerMaker : public framework::OpProtoAndCheckerMaker { AddOutput("IndexTensors", "Index Tensors contains indices for true/false"); AddComment(R"DOC( -Sample dependent Cond Operator: -Given Cond[i] as a 1/0 vector to indicate true/false -The equation is: -Out[i] = subnet_t[i], if Cond[i] == true -Out[i] = subnet_t[i], if Cond[i] == false +Sample Dependent Conditional Operator. + +Given Cond[i] as a 1/0 vector to indicate true/false: +Out[i] = subnet_true[i], if Cond[i] == true +Out[i] = subnet_false[i], if Cond[i] == false + )DOC"); } }; diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..d5b124682d755ffb39f32c9f001a3cf113a01a2c --- /dev/null +++ b/paddle/operators/conditional_block_op.cc @@ -0,0 +1,197 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include +#include "paddle/framework/executor.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class ConditionalOp : public framework::OperatorBase { + public: + ConditionalOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + std::vector InputTensors( + const framework::Scope &scope) const { + std::vector retv; + auto xs = Inputs("X"); + retv.resize(xs.size(), nullptr); + std::transform( + xs.begin(), xs.end(), retv.begin(), + [&scope](const std::string &var_name) -> const framework::LoDTensor * { + auto *var = scope.FindVar(var_name); + PADDLE_ENFORCE(var != nullptr, "Cannot find variable %s", var_name); + return &var->Get(); + }); + return retv; + } +}; + +class ConditionalBlockOp : public ConditionalOp { + public: + ConditionalBlockOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ConditionalOp(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto xs = InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Output("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto *scopes = scope_var->GetMutable>(); + scopes->resize(1); + scopes->front() = &scope.NewScope(); + auto &cur_scope = *scopes->front(); + + auto *block = Attr("block"); + framework::Executor exec(dev_ctx); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + } + } +}; + +class ConditionalBlockOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ConditionalBlockOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The conditional variable of this operator. If X is empty, the " + "whole sub-block will not be executed.") + .AsDuplicable(); + AddInput("Params", "The input variables of the sub-block.").AsDuplicable(); + AddOutput("Out", "The output variables of the sub-block.").AsDuplicable(); + AddOutput("Scope", + "(std::vector) The step scope of conditional block. To " + "unify the conditional block, rnn and while op, the type of " + "scope is std::vector"); + AddAttr( + "block", "The step block of conditional block operator"); + AddComment(R"DOC(Conditional block operator + +Run the sub-block if X is not empty. Params is the other inputs and Out is the +outputs of the sub-block. +)DOC"); + } +}; + +class ConditionalBlockGradOp : public ConditionalOp { + public: + ConditionalBlockGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ConditionalOp(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto xs = this->InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Input("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto &scopes = scope_var->Get>(); + framework::Scope &cur_scope = *scopes[0]; + + auto *block = Attr("block"); + framework::Executor exec(dev_ctx); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + + AssignLocalGradientToGlobal(dev_ctx, cur_scope, Inputs("Params"), + Outputs(framework::GradVarName("Params"))); + + AssignLocalGradientToGlobal(dev_ctx, cur_scope, Inputs("X"), + Outputs(framework::GradVarName("X"))); + } + } + + private: + void AssignLocalGradientToGlobal( + const platform::DeviceContext &dev_ctx, const framework::Scope &cur_scope, + const std::vector &p_names, + const std::vector &pg_names) const { + for (size_t i = 0; i < p_names.size(); ++i) { + auto out_grad_name = pg_names[i]; + auto in_grad_name = framework::GradVarName(p_names[i]); + auto *in_var = cur_scope.FindVar(in_grad_name); + if (in_var == nullptr) { + continue; + } + auto new_in_grad_name = cur_scope.Rename(in_grad_name); + auto assign = + framework::OpRegistry::CreateOp("assign", {{"X", {new_in_grad_name}}}, + {{"Out", {out_grad_name}}}, {}); + assign->Run(cur_scope, dev_ctx); + cur_scope.Rename(new_in_grad_name, in_grad_name); + } + } +}; + +class ConditionalBlockGradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInputs("X")); + if (context->HasInputs("Params")) { + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("Params"))); + context->SetOutputsDim(framework::GradVarName("Params"), + context->GetInputsDim("Params")); + } + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("X"))); + context->SetOutputsDim(framework::GradVarName("X"), + context->GetInputsDim("X")); + } +}; + +class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto grad_op = new framework::OpDescBind(); + grad_op->SetType("conditional_block_grad"); + grad_op->SetInput("X", Input("X")); + grad_op->SetInput("Params", Input("Params")); + grad_op->SetInput("Out", Output("Out")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetInput("Scope", Output("Scope")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params")); + grad_op->SetBlockAttr("block", *this->grad_block_[0]); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(conditional_block, ops::ConditionalBlockOp, + ops::ConditionalBlockOpProtoMaker, + ops::ConditionalBlockGradMaker); +REGISTER_OPERATOR(conditional_block_grad, ops::ConditionalBlockGradOp, + ops::ConditionalBlockGradInferShape); diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc deleted file mode 100644 index 1acb8415d0691df77047806d3c81b51cbb8c59f3..0000000000000000000000000000000000000000 --- a/paddle/operators/conv2d_op.cc +++ /dev/null @@ -1,111 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/conv2d_op.h" - -namespace paddle { -namespace operators { - -void Conv2DOp::InferShape(framework::InferShapeContext* ctx) const { - PADDLE_ENFORCE(ctx->HasInput("Input"), - "Input(Input) of Conv2DOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("Filter"), - "Input(Filter) of Conv2DOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Output"), - "Output(Output) of Conv2DOp should not be null."); - - auto in_dims = ctx->GetInputDim("Input"); - auto filter_dims = ctx->GetInputDim("Filter"); - std::vector strides = ctx->Attrs().Get>("strides"); - std::vector paddings = ctx->Attrs().Get>("paddings"); - int groups = ctx->Attrs().Get("groups"); - int input_channels = in_dims[1]; - int output_channels = filter_dims[0]; - - PADDLE_ENFORCE_EQ(in_dims.size(), 4, "Conv2DOp input should be 4-D."); - PADDLE_ENFORCE_EQ(filter_dims.size(), 4, "Conv2DOp filter should be 4-D."); - PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups, - "The number of input channels should be equal to filter " - "channels * groups."); - PADDLE_ENFORCE_EQ( - output_channels % groups, 0, - "The number of output channels should be divided by groups."); - - auto output_height = - OutputSize(in_dims[2], filter_dims[2], paddings[0], strides[0]); - auto output_width = - OutputSize(in_dims[3], filter_dims[3], paddings[1], strides[1]); - ctx->SetOutputDim("Output", - {in_dims[0], filter_dims[0], output_height, output_width}); -} - -Conv2DOpMaker::Conv2DOpMaker(framework::OpProto* proto, - framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "Input", - "The input tensor of convolution operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of image."); - AddInput("Filter", - "The filter tensor of convolution operator." - "The format of the filter tensor is MCHW, where M is the number of " - "output image channels, C is the number of input image channels, " - "H and W is height and width of filter. " - "If the groups attribute is greater than 1, C equal the number of " - "input image channels divided by the groups."); - AddOutput("Output", - "The output tensor of convolution operator." - "The format of output tensor is also NCHW."); - AddAttr>("strides", "strides of convolution operator.") - .SetDefault({1, 1}); - AddAttr>("paddings", "paddings of convolution operator.") - .SetDefault({0, 0}); - AddAttr( - "groups", - "group size of convolution operator. " - "Refer to grouped convolution in Alex Krizhevsky's paper: " - "when group=2, the first half of the filters are only connected to the " - "first half of the input channels, and the second half only connected " - "to the second half.") - .SetDefault(1); - AddComment(R"DOC( -The convolution operation calculates the output based on the input, filter -and strides, paddings, groups parameters. The size of each dimension of the -parameters is checked in the infer-shape. -)DOC"); -} - -void Conv2DOpGrad::InferShape(framework::InferShapeContext* ctx) const { - auto in_dims = ctx->GetInputDim("Input"); - auto filter_dims = ctx->GetInputDim("Filter"); - if (ctx->HasOutput(framework::GradVarName("Input"))) { - ctx->SetOutputDim(framework::GradVarName("Input"), in_dims); - } - if (ctx->HasOutput(framework::GradVarName("Filter"))) { - ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims); - } -} - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(conv2d, ops::Conv2DOp, ops::Conv2DOpMaker, conv2d_grad, - ops::Conv2DOpGrad); - -REGISTER_OP_CPU_KERNEL( - conv2d, ops::GemmConv2DKernel); -REGISTER_OP_CPU_KERNEL( - conv2d_grad, ops::GemmConvGrad2DKernel); diff --git a/paddle/operators/conv2d_transpose_cudnn_op.cc b/paddle/operators/conv2d_transpose_cudnn_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..fce1357ce5af5f11ccc5941690431393301e6725 --- /dev/null +++ b/paddle/operators/conv2d_transpose_cudnn_op.cc @@ -0,0 +1,50 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/conv_transpose_op.h" + +namespace paddle { +namespace operators { + +class CudnnConv2DTransposeOpMaker : public Conv2DTransposeOpMaker { + public: + CudnnConv2DTransposeOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : Conv2DTransposeOpMaker(proto, op_checker) { + AddAttr>("dilations", "dilations of convolution operator.") + .SetDefault(std::vector{1, 1}); + AddAttr("workspace_size_MB", + "workspace size for cudnn, in MB, " + "workspace is a section of GPU memory which will be " + "allocated/freed each time the operator runs, larger " + "workspace size can increase performance but also requires " + "better hardward. This size should be carefully setted.") + .SetDefault(4096); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(conv2d_transpose_cudnn, ops::ConvTransposeOp, + ops::CudnnConv2DTransposeOpMaker, conv2d_transpose_cudnn_grad, + ops::ConvTransposeOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv2d_transpose_cudnn, + ops::GemmConvTransposeKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_transpose_cudnn_grad, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv2d_transpose_cudnn_op.cu b/paddle/operators/conv2d_transpose_cudnn_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..694526ec01214acf2ec6a3d68d3cf072739ac185 --- /dev/null +++ b/paddle/operators/conv2d_transpose_cudnn_op.cu @@ -0,0 +1,239 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memory.h" +#include "paddle/operators/conv_transpose_op.h" +#include "paddle/platform/assert.h" +#include "paddle/platform/cudnn_helper.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using ScopedTensorDescriptor = platform::ScopedTensorDescriptor; +using ScopedFilterDescriptor = platform::ScopedFilterDescriptor; +using ScopedConvolutionDescriptor = platform::ScopedConvolutionDescriptor; +using DataLayout = platform::DataLayout; + +static constexpr size_t kConvCudnnWorkspaceLimitBytes = 1024 * 1024 * 1024; + +template +class CudnnConvTransposeOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + auto* input = ctx.Input("Input"); + auto* filter = ctx.Input("Filter"); + auto* output = ctx.Output("Output"); + + std::vector strides = ctx.Attr>("strides"); + std::vector paddings = ctx.Attr>("paddings"); + // cudnn v5 does not support dilations + std::vector dilations = ctx.Attr>("dilations"); + int user_workspace_size = ctx.Attr("workspace_size_MB"); + + const T* input_data = input->data(); + const T* filter_data = filter->data(); + T* output_data = output->mutable_data(ctx.GetPlace()); + // ------------------- cudnn descriptors --------------------- + ScopedTensorDescriptor input_desc; + ScopedTensorDescriptor output_desc; + ScopedFilterDescriptor filter_desc; + ScopedConvolutionDescriptor conv_desc; + DataLayout layout = DataLayout::kNCHW; + + // N, M, H, W + cudnnTensorDescriptor_t cudnn_input_desc = input_desc.descriptor( + layout, framework::vectorize2int(input->dims())); + // N, C, O_h, O_w + cudnnTensorDescriptor_t cudnn_output_desc = output_desc.descriptor( + layout, framework::vectorize2int(output->dims())); + // M, C, K_h, K_w + cudnnFilterDescriptor_t cudnn_filter_desc = filter_desc.descriptor( + layout, framework::vectorize2int(filter->dims())); + cudnnConvolutionDescriptor_t cudnn_conv_desc = + conv_desc.descriptor(paddings, strides, dilations); + + // ------------------- cudnn conv workspace --------------------- + void* cudnn_workspace = nullptr; + size_t workspace_size_in_bytes; // final workspace to allocate. + size_t workspace_size_limit = kConvCudnnWorkspaceLimitBytes; + if (user_workspace_size > 0) { + workspace_size_limit = user_workspace_size * 1024 * 1024; + } + // ------------------- cudnn conv algorithm --------------------- + cudnnConvolutionBwdDataAlgo_t algo; + auto handle = ctx.cuda_device_context().cudnn_handle(); + // Get the algorithm + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionBackwardDataAlgorithm( + handle, cudnn_filter_desc, cudnn_input_desc, cudnn_conv_desc, + // dxDesc: Handle to the previously initialized output tensor + // descriptor. + cudnn_output_desc, CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &algo)); + + // get workspace size able to allocate + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardDataWorkspaceSize( + handle, cudnn_filter_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_output_desc, algo, &workspace_size_in_bytes)); + + // Allocate on GPU memory + platform::GPUPlace gpu = boost::get(ctx.GetPlace()); + cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes); + + // ------------------- cudnn conv transpose forward --------------------- + T alpha = 1.0f, beta = 0.0f; + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardData( + handle, &alpha, cudnn_filter_desc, filter_data, cudnn_input_desc, + input_data, cudnn_conv_desc, algo, cudnn_workspace, + workspace_size_in_bytes, &beta, cudnn_output_desc, output_data)); + + // Release the cudnn workspace + paddle::memory::Free(gpu, cudnn_workspace); + } +}; + +template +class CudnnConvTransposeGradOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + auto input = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto output_grad = ctx.Input(framework::GradVarName("Output")); + auto input_grad = ctx.Output(framework::GradVarName("Input")); + auto filter_grad = ctx.Output(framework::GradVarName("Filter")); + const T* input_data = input->data(); + const T* output_grad_data = output_grad->data(); + const T* filter_data = filter->data(); + + std::vector strides = ctx.Attr>("strides"); + std::vector paddings = ctx.Attr>("paddings"); + // cudnn v5 does not support dilations + std::vector dilations = ctx.Attr>("dilations"); + int user_workspace_size = ctx.Attr("workspace_size_MB"); + + // ------------------- cudnn descriptors --------------------- + ScopedTensorDescriptor input_desc; + ScopedTensorDescriptor output_desc; + ScopedFilterDescriptor filter_desc; + ScopedConvolutionDescriptor conv_desc; + DataLayout layout = DataLayout::kNCHW; + + // Input: (N, M, H, W) + cudnnTensorDescriptor_t cudnn_input_desc = input_desc.descriptor( + layout, framework::vectorize2int(input->dims())); + // Output: (N, C, O_H, O_W) + cudnnTensorDescriptor_t cudnn_output_desc = output_desc.descriptor( + layout, framework::vectorize2int(output_grad->dims())); + // Filter (M, C, K_H, K_W) + cudnnFilterDescriptor_t cudnn_filter_desc = filter_desc.descriptor( + layout, framework::vectorize2int(filter->dims())); + + cudnnConvolutionDescriptor_t cudnn_conv_desc = + conv_desc.descriptor(paddings, strides, dilations); + + // ------------------- cudnn backward algorithm --------------------- + cudnnConvolutionFwdAlgo_t data_algo; + cudnnConvolutionBwdFilterAlgo_t filter_algo; + size_t bwd_filter_ws_size, fwd_ws_size; + size_t workspace_size_in_bytes = 0; + size_t workspace_size_limit = kConvCudnnWorkspaceLimitBytes; + if (user_workspace_size > 0) { + workspace_size_limit = user_workspace_size * 1024 * 1024; + } + + auto handle = ctx.cuda_device_context().cudnn_handle(); + if (input_grad) { + // choose backward algorithm for data + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( + handle, cudnn_output_desc, cudnn_filter_desc, cudnn_conv_desc, + cudnn_input_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &data_algo)); + PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardWorkspaceSize( + handle, cudnn_output_desc, cudnn_filter_desc, cudnn_conv_desc, + cudnn_input_desc, data_algo, &fwd_ws_size)); + workspace_size_in_bytes = std::max(workspace_size_in_bytes, fwd_ws_size); + } + + if (filter_grad) { + // choose backward algorithm for filter + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardFilterAlgorithm( + handle, cudnn_output_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_filter_desc, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &filter_algo)); + + // get workspace for backwards filter algorithm + PADDLE_ENFORCE( + platform::dynload::cudnnGetConvolutionBackwardFilterWorkspaceSize( + handle, cudnn_output_desc, cudnn_input_desc, cudnn_conv_desc, + cudnn_filter_desc, filter_algo, &bwd_filter_ws_size)); + workspace_size_in_bytes = + std::max(workspace_size_in_bytes, bwd_filter_ws_size); + } + + // ------------------- cudnn conv workspace --------------------- + // Already on GPU + void* cudnn_workspace = nullptr; + platform::GPUPlace gpu = boost::get(ctx.GetPlace()); + cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes); + // ------------------- cudnn conv backward data --------------------- + // FIXME(typhoonzero): template type T may not be the same as cudnn call. + T alpha = 1.0f, beta = 0.0f; + if (input_grad) { + T* input_grad_data = input_grad->mutable_data(ctx.GetPlace()); + auto t = framework::EigenVector::Flatten(*input_grad); + t.device(ctx.GetEigenDevice()) = + t.constant(static_cast(0)); + + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionForward( + handle, &alpha, cudnn_output_desc, output_grad_data, + cudnn_filter_desc, filter_data, cudnn_conv_desc, data_algo, + cudnn_workspace, workspace_size_in_bytes, &beta, cudnn_input_desc, + input_grad_data)); + } + + // ------------------- cudnn conv backward filter --------------------- + if (filter_grad) { + T* filter_grad_data = filter_grad->mutable_data(ctx.GetPlace()); + auto t = framework::EigenVector::Flatten(*filter_grad); + t.device(ctx.GetEigenDevice()) = + t.constant(static_cast(0)); + // Gradient with respect to the filter + PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardFilter( + handle, &alpha, cudnn_output_desc, output_grad_data, cudnn_input_desc, + input_data, cudnn_conv_desc, filter_algo, cudnn_workspace, + workspace_size_in_bytes, &beta, cudnn_filter_desc, filter_grad_data)); + } + // Release the cudnn workspace + paddle::memory::Free(gpu, cudnn_workspace); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn, + ops::CudnnConvTransposeOpKernel); +REGISTER_OP_GPU_KERNEL(conv2d_transpose_cudnn_grad, + ops::CudnnConvTransposeGradOpKernel); diff --git a/paddle/operators/conv2dtranspose_op.cc b/paddle/operators/conv2dtranspose_op.cc deleted file mode 100644 index c1b231906e2f172b6f9cee55f850d1a5ec6c3221..0000000000000000000000000000000000000000 --- a/paddle/operators/conv2dtranspose_op.cc +++ /dev/null @@ -1,107 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/conv2dtranspose_op.h" - -namespace paddle { -namespace operators { - -void Conv2DTransposeOp::InferShape(framework::InferShapeContext* ctx) const { - PADDLE_ENFORCE(ctx->HasInput("Input"), - "Input(Input) of Conv2DTransposeOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("Filter"), - "Input(Filter) of Conv2DTransposeOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Output"), - "Output(Output) of Conv2DTransposeOp should not be null."); - - auto in_dims = ctx->GetInputDim("Input"); - auto filter_dims = ctx->GetInputDim("Filter"); - std::vector strides = ctx->Attrs().Get>("strides"); - std::vector paddings = ctx->Attrs().Get>("paddings"); - - for (size_t i = 0; i < paddings.size(); ++i) { - PADDLE_ENFORCE_EQ(paddings[i], 0, - "No Padding allowed in conv transpose op."); - } - - PADDLE_ENFORCE_EQ(in_dims.size(), 4, - "Conv2DTransposeOp input should be 4-D tensor."); - PADDLE_ENFORCE_EQ(filter_dims.size(), 4, - "Conv2DTransposeOp filter should be 4-D tensor."); - PADDLE_ENFORCE_EQ(in_dims[1], filter_dims[0], - "input and kernel input dimension should be equal."); - - auto output_height = (in_dims[2] - 1) * strides[0] + filter_dims[2]; - auto output_width = (in_dims[3] - 1) * strides[1] + filter_dims[3]; - ctx->SetOutputDim("Output", - {in_dims[0], filter_dims[1], output_height, output_width}); -} - -Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( - framework::OpProto* proto, framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "Input", - "(Tensor) The input tensor of convolution transpose operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of input channels, H and W is the height and width of image."); - AddInput("Filter", - "(Tensor) The filter tensor of convolution transpose operator." - "The format of the filter tensor is CMHW, where C is the number of " - "output image channels, M is the number of input image channels, " - "H and W is height and width of filter. " - "We enforce groups number == 1 and padding == 0 in " - "convolution transpose Scenario."); - AddOutput("Output", - "(Tensor) The output tensor of convolution transpose operator." - "The format of output tensor is also NCHW."); - AddAttr>("strides", - "strides of convolution transpose operator.") - .SetDefault({1, 1}); - AddAttr>("paddings", - "paddings of convolution transpose operator.") - .SetDefault({0, 0}); - AddComment(R"DOC( -The convolution transpose operation calculates the output based on the input, filter -and strides, paddings, groups parameters. The size of each dimension of the -parameters is checked in the infer-shape. -)DOC"); -} - -void Conv2DTransposeOpGrad::InferShape( - framework::InferShapeContext* ctx) const { - auto in_dims = ctx->GetInputDim("Input"); - auto filter_dims = ctx->GetInputDim("Filter"); - if (ctx->HasOutput(framework::GradVarName("Input"))) { - ctx->SetOutputDim(framework::GradVarName("Input"), in_dims); - } - if (ctx->HasOutput(framework::GradVarName("Filter"))) { - ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims); - } -} - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(conv2dtranspose, ops::Conv2DTransposeOp, - ops::Conv2DTransposeOpMaker, conv2dtranspose_grad, - ops::Conv2DTransposeOpGrad); - -REGISTER_OP_CPU_KERNEL( - conv2dtranspose, - ops::GemmConv2DTransposeKernel); -REGISTER_OP_CPU_KERNEL( - conv2dtranspose_grad, - ops::GemmConv2DTransposeGradKernel); diff --git a/paddle/operators/conv2dtranspose_op.h b/paddle/operators/conv2dtranspose_op.h deleted file mode 100644 index 8c70b3dcec1e26ab3d8a42d88040764c643b5ae6..0000000000000000000000000000000000000000 --- a/paddle/operators/conv2dtranspose_op.h +++ /dev/null @@ -1,254 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" -#include "paddle/operators/math/im2col.h" -#include "paddle/operators/math/math_function.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -using DDim = framework::DDim; - -// Define Op classes in .h file so that other conv transpose -// operator implementations can reuse the code. -class Conv2DTransposeOpMaker : public framework::OpProtoAndCheckerMaker { - public: - Conv2DTransposeOpMaker(framework::OpProto* proto, - framework::OpAttrChecker* op_checker); -}; - -class Conv2DTransposeOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(framework::InferShapeContext* ctx) const override; -}; - -class Conv2DTransposeOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(framework::InferShapeContext* ctx) const override; -}; - -template -class GemmConv2DTransposeKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - const Tensor* input = context.Input("Input"); - // The filter will be reshaped, so it should not be constant pointer - Tensor filter = *context.Input("Filter"); - - Tensor* output = context.Output("Output"); - - std::vector strides = context.Attr>("strides"); - - // TODO(Zhuoyuan): Paddings can be added in future. - // groups will alway be disabled in conv2dtranspose. - - const int batch_size = input->dims()[0]; - const int m = input->dims()[1]; - const int h = input->dims()[2]; - const int w = input->dims()[3]; - - const int k_h = filter.dims()[2]; - const int k_w = filter.dims()[3]; - - const int c = output->dims()[1]; // output channels - const int o_h = output->dims()[2]; - const int o_w = output->dims()[3]; - - paddle::operators::math::Col2ImFunctor< - paddle::operators::math::ColFormat::kCFO, Place, T> - col2im; - - // use col_shape in the im2col and col2im calculation - DDim col_shape = {c, k_h, k_w, h, w}; - - // use col_matrix_shape in the gemm calculation - DDim col_matrix_shape = {c * k_h * k_w, h * w}; - - Tensor col; - col.mutable_data(col_shape, context.GetPlace()); - // col_matrix shares the same piece of data with col, - // but will be reshaped into a two-dimensional matrix shape - // to call the matrix multiplication interface. - Tensor col_matrix; - col_matrix.ShareDataWith(col); - col_matrix.Resize(col_matrix_shape); - - DDim output_shape = {c, o_h, o_w}; - DDim input_matrix_shape = {m, h * w}; - - DDim filter_matrix_shape = {m, c * k_h * k_w}; - filter.Resize(filter_matrix_shape); - - // convolution transpose: gemm + col2im (similar to conv-backward on input) - - output->mutable_data(context.GetPlace()); - auto t = framework::EigenVector::Flatten(*output); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); - - for (int i = 0; i < batch_size; i++) { - // batch with size (M, h * w) - Tensor input_batch = input->Slice(i, i + 1).Resize(input_matrix_shape); - // filter size: (M, c * k_h * k_w) - - // output size: (c, o_h, o_w) - Tensor output_batch = output->Slice(i, i + 1).Resize(output_shape); - - // col_matrix = filter * input_batch - // of shape (c * k_h * k_w, h * w) - math::matmul(context.device_context(), filter, true, - input_batch, false, T(1.0), &col_matrix, T(0.0)); - col2im(context.device_context(), output_batch, col, strides[0], - strides[1], 0, 0, 0, 0); - } - } -}; - -template -class GemmConv2DTransposeGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - const Tensor* input = context.Input("Input"); - const Tensor* output_grad = - context.Input(framework::GradVarName("Output")); - - // For filter, we do not use const pointer b/c we will do reshape, - // but we should avoid modifying its value. - Tensor filter = *context.Input("Filter"); - - Tensor* input_grad = - context.Output(framework::GradVarName("Input")); - Tensor* filter_grad = - context.Output(framework::GradVarName("Filter")); - - std::vector strides = context.Attr>("strides"); - // Actually, no paddings and groups allowed in conv transpose. - std::vector paddings = context.Attr>("paddings"); - - const int batch_size = input->dims()[0]; - const int m = input->dims()[1]; - const int h = input->dims()[2]; - const int w = input->dims()[3]; - - const int k_h = filter.dims()[2]; - const int k_w = filter.dims()[3]; - - const int c = output_grad->dims()[1]; // output channels - const int o_h = output_grad->dims()[2]; - const int o_w = output_grad->dims()[3]; - - // Only im2col functor required for bp to get to the right shape - paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kCFO, Place, T> - im2col; - - // use col_shape in the im2col and col2im calculation - DDim col_shape = {c, k_h, k_w, h, w}; - - // use col_matrix_shape in the gemm calculation - DDim col_matrix_shape_f = {c * h * w, k_h * k_w}; - - Tensor col; - col.mutable_data(col_shape, context.GetPlace()); - // col_matrix shares the same piece of data with col, - // but will be reshaped into a two-dimensional matrix shape - // to call the matrix multiplication interface. - - DDim output_shape = {c, o_h, o_w}; - DDim input_matrix_shape = {m, h * w}; - - DDim filter_matrix_shape = {m, c * k_h * k_w}; - filter.Resize(filter_matrix_shape); - - // convolution transpose grad on input: - // im2col + gemm (similar to conv-forward) - // input need to compute gradient - if (input_grad) { - Tensor col_matrix; - col_matrix.ShareDataWith(col); - DDim col_matrix_shape = {c * k_h * k_w, h * w}; - col_matrix.Resize(col_matrix_shape); - - input_grad->mutable_data(context.GetPlace()); - auto t = framework::EigenVector::Flatten(*input_grad); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); - - for (int i = 0; i < batch_size; i++) { - // batch with size (c, o_h * o_w) - Tensor output_grad_batch = - output_grad->Slice(i, i + 1).Resize(output_shape); - // filter of size (m, c * k_h * k_w) - - // batch with size (m, h, w) - Tensor input_grad_batch = - input_grad->Slice(i, i + 1).Resize(input_matrix_shape); - - // im2col: dy from (c, o_h, o_w) -> (c * k_h * k_w, h * w) - im2col(context.device_context(), output_grad_batch, col, strides[0], - strides[1], paddings[0], paddings[0], paddings[1], paddings[1]); - - // gemm: dx = filter * dy - // (m, c * k_h * k_w) * (c * k_h * k_w, h * w) -> (m, c, h) - math::matmul(context.device_context(), filter, false, - col_matrix, false, T(1.0), &input_grad_batch, - T(0.0)); - } - } - - // filter gradient required - if (filter_grad) { - Tensor col_matrix_f; - col_matrix_f.ShareDataWith(col); - DDim col_matrix_shape_f = {c * h * w, k_h * k_w}; - col_matrix_f.Resize(col_matrix_shape_f); - - filter_grad->mutable_data(context.GetPlace()); - Tensor filter_grad_ = *filter_grad; - filter_grad_.Resize(filter_matrix_shape); - auto t = framework::EigenVector::Flatten(filter_grad_); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); - - for (int i = 0; i < batch_size; ++i) { - // batch with size (c, o_h, o_w) - Tensor output_grad_batch = - output_grad->Slice(i, i + 1).Resize(output_shape); - // input batch - Tensor in_batch = input->Slice(i, i + 1).Resize(input_matrix_shape); - - // im2col: (c * h * w, k_h * k_w) - im2col(context.device_context(), output_grad_batch, col, strides[0], - strides[1], paddings[0], paddings[0], paddings[1], paddings[1]); - - // gemm: d_filter = x * y_grad^T - // (m, c * h * w) * (k_h * k_w, c * h * w) -> (m, c, h) - math::matmul(context.device_context(), in_batch, false, - col_matrix_f, true, T(1.0), &filter_grad_, - T(1.0)); - } - } - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/conv_cudnn_op.cc b/paddle/operators/conv_cudnn_op.cc index 4288f300dd5b0464f2b4394cdb0b44f93060ae74..97f31bf22d7072d89bd043045045dcb5bb5518b8 100644 --- a/paddle/operators/conv_cudnn_op.cc +++ b/paddle/operators/conv_cudnn_op.cc @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/conv2d_op.h" +#include "paddle/operators/conv_op.h" namespace paddle { namespace operators { @@ -29,7 +29,7 @@ class CudnnConvOpMaker : public Conv2DOpMaker { "workspace is a section of GPU memory which will be " "allocated/freed each time the operator runs, larger " "workspace size can increase performance but also requires " - "better hardward. This size should be carefully setted.") + "better hardware. This size should be chosen carefully.") .SetDefault(4096); } }; @@ -38,10 +38,11 @@ class CudnnConvOpMaker : public Conv2DOpMaker { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(conv_cudnn, ops::Conv2DOp, ops::CudnnConvOpMaker, conv_cudnn_grad, - ops::Conv2DOpGrad); -REGISTER_OP_CPU_KERNEL( - conv_cudnn, ops::GemmConv2DKernel); +REGISTER_OP(conv_cudnn, ops::ConvOp, ops::CudnnConvOpMaker, conv_cudnn_grad, + ops::ConvOpGrad); + +REGISTER_OP_CPU_KERNEL(conv_cudnn, + ops::GemmConvKernel); REGISTER_OP_CPU_KERNEL( conv_cudnn_grad, - ops::GemmConvGrad2DKernel); + ops::GemmConvGradKernel); diff --git a/paddle/operators/conv_cudnn_op.cu b/paddle/operators/conv_cudnn_op.cu index e2eb157f40c0039f87c41d28f8732cd4901a046d..2aec4a2760260623c4c7054c590afa8e1c6c3fea 100644 --- a/paddle/operators/conv_cudnn_op.cu +++ b/paddle/operators/conv_cudnn_op.cu @@ -15,7 +15,7 @@ #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/memory/memory.h" -#include "paddle/operators/conv2d_op.h" +#include "paddle/operators/conv_op.h" #include "paddle/platform/assert.h" #include "paddle/platform/cudnn_helper.h" @@ -27,7 +27,6 @@ using ScopedTensorDescriptor = platform::ScopedTensorDescriptor; using ScopedFilterDescriptor = platform::ScopedFilterDescriptor; using ScopedConvolutionDescriptor = platform::ScopedConvolutionDescriptor; using DataLayout = platform::DataLayout; -using CUDADeviceContext = platform::CUDADeviceContext; static constexpr size_t kCONV_CUDNN_WORKSPACE_LIMIT_BYTES = 1024 * 1024 * 1024; diff --git a/paddle/operators/conv_op.cc b/paddle/operators/conv_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6f65f10165929316f971d195f3790fd9e7ed376 --- /dev/null +++ b/paddle/operators/conv_op.cc @@ -0,0 +1,209 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/conv_op.h" + +namespace paddle { +namespace operators { + +void ConvOp::InferShape(framework::InferShapeContext* ctx) const { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(Input) of ConvOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Filter"), + "Input(Filter) of ConvOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Output"), + "Output(Output) of ConvOp should not be null."); + + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + std::vector strides = ctx->Attrs().Get>("strides"); + std::vector paddings = ctx->Attrs().Get>("paddings"); + int groups = ctx->Attrs().Get("groups"); + int input_channels = in_dims[1]; + int output_channels = filter_dims[0]; + + PADDLE_ENFORCE(in_dims.size() == 4 || in_dims.size() == 5, + "Conv intput should be 4-D or 5-D tensor."); + PADDLE_ENFORCE_EQ( + in_dims.size(), filter_dims.size(), + "Conv input dimension and filter dimension should be the same."); + PADDLE_ENFORCE( + in_dims.size() - strides.size() == 2U, + "Conv input dimension and strides dimension should be consistent."); + PADDLE_ENFORCE_EQ( + paddings.size(), strides.size(), + "Conv paddings dimension and Conv strides dimension should be the same."); + PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups, + "The number of input channels should be equal to filter " + "channels * groups."); + PADDLE_ENFORCE_EQ( + output_channels % groups, 0, + "The number of output channels should be divided by groups."); + + std::vector output_shape({in_dims[0], filter_dims[0]}); + for (size_t i = 0; i < paddings.size(); ++i) { + output_shape.push_back(OutputSize(in_dims[i + 2], filter_dims[i + 2], + paddings[i], strides[i])); + } + ctx->SetOutputDim("Output", framework::make_ddim(output_shape)); +} + +Conv2DOpMaker::Conv2DOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "(Tensor) The input tensor of convolution operator. " + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of channels, H is the height of the feature, " + "and W is the width of the feature."); + AddInput("Filter", + "(Tensor) The filter tensor of convolution operator. " + "The format of the filter tensor is MCHW, where M is the number of " + "output image channels, C is the number of input image channels, " + "H is the height of the filter, and W is the width of the filter. " + "If the groups attribute is greater than 1, C equals the number of " + "input image channels divided by the groups."); + AddOutput("Output", + "(Tensor) The output tensor of convolution operator. " + "The format of output tensor is also NCHW."); + AddAttr>("strides", "strides of convolution operator.") + .SetDefault({1, 1}); + AddAttr>("paddings", "paddings of convolution operator.") + .SetDefault({0, 0}); + AddAttr( + "groups", + "(int default:1), the group size of convolution operator. " + "According to grouped convolution in Alex Krizhevsky's Deep CNN paper: " + "when group=2, the first half of the filters is only connected to the " + "first half of the input channels, while the second half of the filters " + "is only connected to the second half of the input channels.") + .SetDefault(1); + AddComment(R"DOC( +Convolution Operator. + +The convolution operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. +Input(Input, Filter) and output(Output) are in NCHW format. Where N is batch +size, C is the number of channels, H is the height of the feature, and W is +the width of the feature. Parameters(ksize, strides, paddings) are two elements. +These two elements represent height and width, respectively. +The input(X) size and output(Out) size may be different. + +Example: + Input: + Input shape: (N, C_in, H_in, W_in) + Filter shape: (C_out, C_in, H_f, W_f) + Output: + Output shape: (N, C_out, H_out, W_out) + where + H_out = (H_in - filter_size[0] + 2 * paddings[0]) / strides[0] + 1; + W_out = (W_in - filter_size[1] + 2 * paddings[1]) / strides[1] + 1; +)DOC"); +} + +Conv3DOpMaker::Conv3DOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "(Tensor) The input tensor of convolution operator. " + "The format of input tensor is NCDHW. Where N is batch size, C is the " + "number of channels, D is the depth of the feature, H is the height of " + "the feature, " + "and W is the width of the feature."); + AddInput("Filter", + "(Tensor) The filter tensor of convolution operator. " + "The format of the filter tensor is MCDHW, where M is the number of " + "output image channels, C is the number of input image channels, " + "D is the depth of the filter, H is the height of the filter, and W " + "is the width of the filter." + "If the groups attribute is greater than 1, C equals the number of " + "input image channels divided by the groups."); + AddOutput("Output", + "(Tensor) The output tensor of convolution operator." + "The format of output tensor is also NCDHW."); + AddAttr>( + "strides", + "(vector, default:{0, 0, 0}), the strides of convolution operator.") + .SetDefault({1, 1, 1}); + AddAttr>( + "paddings", + "(vector, default:{0, 0, 0}), the paddings of convolution operator.") + .SetDefault({0, 0, 0}); + AddAttr( + "groups", + "(int default:1), the group size of convolution operator. " + "According to grouped convolution in Alex Krizhevsky's Deep CNN paper: " + "when group=2, the first half of the filters is only connected to the " + "first half of the input channels, while the second half of the filters " + "is only connected to the second half of the input channels.") + .SetDefault(1); + + AddComment(R"DOC( +Convolution3D Operator. + +The convolution operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. +Input(Input, Filter) and output(Output) are in NCDHW format. Where N is batch +size, C is the number of channels,D is the depth of the feature, H is the height of +the feature, and W is the width of the feature. Parameters(ksize, strides, paddings) +are three elements. These three elements represent depth, height and width, respectively. +The input(X) size and output(Out) size may be different. + +Example: + Input: + Input shape: (N, C_in, D_in, H_in, W_in) + Filter shape: (C_out, C_in, D_f, H_f, W_f) + Output: + Output shape: (N, C_out, D_out, H_out, W_out) + where + D_out = (D_in - filter_size[0] + 2 * paddings[0]) / strides[0] + 1; + H_out = (H_in - filter_size[1] + 2 * paddings[1]) / strides[1] + 1; + W_out = (W_in - filter_size[2] + 2 * paddings[2]) / strides[2] + 1; +)DOC"); +} + +void ConvOpGrad::InferShape(framework::InferShapeContext* ctx) const { + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + if (ctx->HasOutput(framework::GradVarName("Input"))) { + ctx->SetOutputDim(framework::GradVarName("Input"), in_dims); + } + if (ctx->HasOutput(framework::GradVarName("Filter"))) { + ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims); + } +} + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(conv2d, ops::ConvOp, ops::Conv2DOpMaker, conv2d_grad, + ops::ConvOpGrad); +namespace ops = paddle::operators; +REGISTER_OP(conv3d, ops::ConvOp, ops::Conv3DOpMaker, conv3d_grad, + ops::ConvOpGrad); + +REGISTER_OP_CPU_KERNEL(conv2d, + ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_grad, ops::GemmConvGradKernel); + +REGISTER_OP_CPU_KERNEL(conv3d, + ops::GemmConvKernel); +REGISTER_OP_CPU_KERNEL( + conv3d_grad, ops::GemmConvGradKernel); diff --git a/paddle/operators/conv2d_op.cu b/paddle/operators/conv_op.cu similarity index 64% rename from paddle/operators/conv2d_op.cu rename to paddle/operators/conv_op.cu index c697c9466d34c29af6976f3a4d2d0a24ba778ceb..8e6f9da455b7291049aee57189dae15b8bcc2150 100644 --- a/paddle/operators/conv2d_op.cu +++ b/paddle/operators/conv_op.cu @@ -12,11 +12,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/conv2d_op.h" +#include "paddle/operators/conv_op.h" namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(conv2d, + ops::GemmConvKernel); REGISTER_OP_GPU_KERNEL( - conv2d, ops::GemmConv2DKernel); + conv2d_grad, ops::GemmConvGradKernel); + +REGISTER_OP_GPU_KERNEL(conv3d, + ops::GemmConvKernel); REGISTER_OP_GPU_KERNEL( - conv2d_grad, ops::GemmConvGrad2DKernel); + conv3d_grad, ops::GemmConvGradKernel); diff --git a/paddle/operators/conv2d_op.h b/paddle/operators/conv_op.h similarity index 54% rename from paddle/operators/conv2d_op.h rename to paddle/operators/conv_op.h index 0621389a79eee6b5e75b1eab309b49f8aa4a97ca..7c1729213bf3f5f3987afbf2d51d5b5339ae521d 100644 --- a/paddle/operators/conv2d_op.h +++ b/paddle/operators/conv_op.h @@ -18,6 +18,7 @@ limitations under the License. */ #include "paddle/framework/op_registry.h" #include "paddle/operators/math/im2col.h" #include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/vol2col.h" namespace paddle { namespace operators { @@ -40,14 +41,20 @@ class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker); }; -class Conv2DOp : public framework::OperatorWithKernel { +class Conv3DOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Conv3DOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker); +}; + +class ConvOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override; }; -class Conv2DOpGrad : public framework::OperatorWithKernel { +class ConvOpGrad : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -55,7 +62,7 @@ class Conv2DOpGrad : public framework::OperatorWithKernel { }; template -class GemmConv2DKernel : public framework::OpKernel { +class GemmConvKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { const Tensor* input = context.Input("Input"); @@ -70,51 +77,78 @@ class GemmConv2DKernel : public framework::OpKernel { std::vector paddings = context.Attr>("paddings"); int groups = context.Attr("groups"); - int batch_size = input->dims()[0]; - int input_channels = input->dims()[1]; - int filter_height = filter.dims()[filter.dims().size() - 2]; - int filter_width = filter.dims()[filter.dims().size() - 1]; - int output_channels = output->dims()[1]; - int output_height = output->dims()[2]; - int output_width = output->dims()[3]; - - paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kCFO, Place, T> - im2col; + const int batch_size = static_cast(input->dims()[0]); + + // filter_shape_vec: {k_h, k_w} or {k_d, k_h, k_w} + std::vector filter_shape_vec(framework::vectorize(filter.dims())); + filter_shape_vec.erase(filter_shape_vec.begin(), + filter_shape_vec.begin() + 2); + + // output_shape_vec: {o_h, o_w} or {o_d, o_h, o_w} + std::vector output_shape_vec(framework::vectorize(output->dims())); + output_shape_vec.erase(output_shape_vec.begin(), + output_shape_vec.begin() + 2); + // use col_shape in the im2col calculation - framework::DDim col_shape = {input_channels / groups, filter_height, - filter_width, output_height, output_width}; + // col_shape_vec: {i_c/g, k_h, k_w, o_h, o_w} or {i_c/g, k_d, k_h, k_w, o_d, + // o_h, o_w} + std::vector col_shape_vec; + col_shape_vec.push_back(input->dims()[1] / groups); + col_shape_vec.insert(col_shape_vec.end(), filter_shape_vec.begin(), + filter_shape_vec.end()); + col_shape_vec.insert(col_shape_vec.end(), output_shape_vec.begin(), + output_shape_vec.end()); + framework::DDim col_shape(framework::make_ddim(col_shape_vec)); + // use col_matrix_shape in the gemm calculation - framework::DDim col_matrix_shape = { - input_channels / groups * filter_height * filter_width, - output_height * output_width}; + // size: (i_c/g * k_h * k_w, o_h * o_w) or (i_c/g * k_d * k_h * k_w, o_d * + // o_h * o_w) + framework::DDim col_matrix_shape = + framework::flatten_to_2d(col_shape, filter_shape_vec.size() + 1); + Tensor col; col.mutable_data(col_shape, context.GetPlace()); // col_matrix shares the same piece of data with col, // but will be reshaped into a two-dimensional matrix shape // to call the matrix multiplication interface. - Tensor col_matrix = col; + Tensor col_matrix; + col_matrix.ShareDataWith(col); col_matrix.Resize(col_matrix_shape); - framework::DDim input_shape = {input->dims()[1], input->dims()[2], - input->dims()[3]}; + framework::DDim input_shape = framework::slice_ddim( + input->dims(), 1, static_cast(input->dims().size())); + framework::DDim filter_matrix_shape = {filter.dims()[0], filter.numel() / filter.dims()[0]}; filter.Resize(filter_matrix_shape); - framework::DDim output_matrix_shape = {output_channels, - output_height * output_width}; - // convolution operator: im2col + gemm - int in_step = input_channels / groups; - int out_step = output_channels / groups; + framework::DDim output_matrix_shape = { + output->dims()[1], + output->numel() / (output->dims()[0] * output->dims()[1])}; + + // convolution operator: im2col(or vol2col) + gemm + int in_step = static_cast(input->dims()[1]) / groups; + int out_step = static_cast(output->dims()[1]) / groups; + for (int i = 0; i < batch_size; i++) { Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); Tensor out_batch = output->Slice(i, i + 1).Resize(output_matrix_shape); for (int g = 0; g < groups; g++) { - // im2col Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); - im2col(context.device_context(), in_slice, col, strides[0], strides[1], - paddings[0], paddings[0], paddings[1], paddings[1]); + + if (filter_shape_vec.size() == 2) { + // im2col + math::Im2ColFunctor im2col; + im2col(context.device_context(), in_slice, col, strides[0], + strides[1], paddings[0], paddings[0], paddings[1], + paddings[1]); + } else if (filter_shape_vec.size() == 3) { + // vol2col + math::Vol2ColFunctor vol2col; + vol2col(context.device_context(), in_slice, col, strides[0], + strides[1], strides[2], paddings[0], paddings[1], + paddings[2]); + } // gemm Tensor out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step); @@ -127,7 +161,7 @@ class GemmConv2DKernel : public framework::OpKernel { }; template -class GemmConvGrad2DKernel : public framework::OpKernel { +class GemmConvGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { const Tensor* input = context.Input("Input"); @@ -137,64 +171,79 @@ class GemmConvGrad2DKernel : public framework::OpKernel { context.Output(framework::GradVarName("Input")); Tensor* filter_grad = context.Output(framework::GradVarName("Filter")); - // The filter and filter_grad will be reshaped in the calculations, // so here use an assignment operation, // that avoids modifying the variable in the Scope. Tensor filter = *context.Input("Filter"); + if (!input_grad && !filter_grad) return; + std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); int groups = context.Attr("groups"); - int batch_size = input->dims()[0]; - int input_channels = input->dims()[1]; - int filter_height = filter.dims()[filter.dims().size() - 2]; - int filter_width = filter.dims()[filter.dims().size() - 1]; - int output_channels = output_grad->dims()[1]; - int output_height = output_grad->dims()[2]; - int output_width = output_grad->dims()[3]; - - paddle::operators::math::Col2ImFunctor< - paddle::operators::math::ColFormat::kCFO, Place, T> - col2im; - paddle::operators::math::Im2ColFunctor< - paddle::operators::math::ColFormat::kCFO, Place, T> - im2col; - // use col_shape in the im2col and col2im calculation - framework::DDim col_shape = {input_channels / groups, filter_height, - filter_width, output_height, output_width}; + const int batch_size = static_cast(input->dims()[0]); + + // filter_shape_vec: {k_h, k_w} or {k_d, k_h, k_w} + std::vector filter_shape_vec(framework::vectorize(filter.dims())); + filter_shape_vec.erase(filter_shape_vec.begin(), + filter_shape_vec.begin() + 2); + + // output_shape_vec: {o_h, o_w} or {o_d, o_h, o_w} + std::vector output_shape_vec( + framework::vectorize(output_grad->dims())); + output_shape_vec.erase(output_shape_vec.begin(), + output_shape_vec.begin() + 2); + + // use col_shape in the im2col calculation + // col_shape_vec: {i_c/g, k_h, k_w, o_h, o_w} or {i_c/g, k_d, k_h, k_w, o_d, + // o_h, o_w} + std::vector col_shape_vec; + col_shape_vec.push_back(input->dims()[1] / groups); + col_shape_vec.insert(col_shape_vec.end(), filter_shape_vec.begin(), + filter_shape_vec.end()); + col_shape_vec.insert(col_shape_vec.end(), output_shape_vec.begin(), + output_shape_vec.end()); + framework::DDim col_shape(framework::make_ddim(col_shape_vec)); + // use col_matrix_shape in the gemm calculation - framework::DDim col_matrix_shape = { - input_channels / groups * filter_height * filter_width, - output_height * output_width}; - Tensor col; - col.mutable_data(col_shape, context.GetPlace()); - // col_matrix shares the same piece of data with col, - // but will be reshaped into a two-dimensional matrix shape - // to call the matrix multiplication interface. - Tensor col_matrix = col; - col_matrix.Resize(col_matrix_shape); + // size: (i_c/g * k_h * k_w, o_h * o_w) + // or + // (i_c/g * k_d * k_h * k_w, o_d * o_h * o_w) + framework::DDim col_matrix_shape = + framework::flatten_to_2d(col_shape, filter_shape_vec.size() + 1); - framework::DDim input_shape = {input->dims()[1], input->dims()[2], - input->dims()[3]}; - framework::DDim output_matrix_shape = { - output_grad->dims()[1], - output_grad->dims()[2] * output_grad->dims()[3]}; + framework::DDim input_shape = framework::slice_ddim( + input->dims(), 1, static_cast(input->dims().size())); framework::DDim filter_matrix_shape = {filter.dims()[0], filter.numel() / filter.dims()[0]}; filter.Resize(filter_matrix_shape); - // convolution backward input operator: gemm + col2im - // convolution backward weight operator: im2col + gemm - int in_step = input_channels / groups; - int out_step = output_channels / groups; + framework::DDim output_matrix_shape = { + output_grad->dims()[1], + output_grad->numel() / + (output_grad->dims()[0] * output_grad->dims()[1])}; + + // convolution backward input operator: gemm + col2im(or col2vol) + // convolution backward weight operator: im2col(or vol2col) + gemm + int in_step = static_cast(input->dims()[1]) / groups; + int out_step = static_cast(output_grad->dims()[1]) / groups; + + Tensor col; + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix; + col.mutable_data(col_shape, context.GetPlace()); + col_matrix.ShareDataWith(col); + col_matrix.Resize(col_matrix_shape); + + math::SetConstant set_zero; if (input_grad) { input_grad->mutable_data(context.GetPlace()); - auto t = framework::EigenVector::Flatten(*input_grad); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + set_zero(context.device_context(), input_grad, static_cast(0)); for (int i = 0; i < batch_size; i++) { Tensor out_grad_batch = @@ -208,13 +257,22 @@ class GemmConvGrad2DKernel : public framework::OpKernel { math::matmul(context.device_context(), filter_slice, true, out_grad_slice, false, T(1.0), &col_matrix, T(0.0)); - // col2im Tensor in_grad_slice = in_grad_batch.Slice(g * in_step, (g + 1) * in_step); - col2im(context.device_context(), in_grad_slice, col, strides[0], - strides[1], paddings[0], paddings[0], paddings[1], - paddings[1]); + + if (filter_shape_vec.size() == 2) { + math::Col2ImFunctor col2im; + col2im(context.device_context(), in_grad_slice, col, strides[0], + strides[1], paddings[0], paddings[0], paddings[1], + paddings[1]); + + } else if (filter_shape_vec.size() == 3) { + math::Col2VolFunctor col2vol; + col2vol(context.device_context(), in_grad_slice, col, strides[0], + strides[1], strides[2], paddings[0], paddings[1], + paddings[2]); + } } } } @@ -223,8 +281,7 @@ class GemmConvGrad2DKernel : public framework::OpKernel { filter_grad->mutable_data(context.GetPlace()); Tensor filter_grad_ = *filter_grad; filter_grad_.Resize(filter_matrix_shape); - auto t = framework::EigenVector::Flatten(filter_grad_); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + set_zero(context.device_context(), filter_grad, static_cast(0)); for (int i = 0; i < batch_size; i++) { Tensor out_grad_batch = @@ -235,9 +292,18 @@ class GemmConvGrad2DKernel : public framework::OpKernel { Tensor out_grad_slice = out_grad_batch.Slice(g * out_step, (g + 1) * out_step); Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); - im2col(context.device_context(), in_slice, col, strides[0], - strides[1], paddings[0], paddings[0], paddings[1], - paddings[1]); + + if (filter_shape_vec.size() == 2) { + math::Im2ColFunctor im2col; + im2col(context.device_context(), in_slice, col, strides[0], + strides[1], paddings[0], paddings[0], paddings[1], + paddings[1]); + } else if (filter_shape_vec.size() == 3) { + math::Vol2ColFunctor vol2col; + vol2col(context.device_context(), in_slice, col, strides[0], + strides[1], strides[2], paddings[0], paddings[1], + paddings[2]); + } // gemm Tensor filter_grad_slice = @@ -250,6 +316,5 @@ class GemmConvGrad2DKernel : public framework::OpKernel { } } }; - } // namespace operators } // namespace paddle diff --git a/paddle/operators/conv_shift_op.cc b/paddle/operators/conv_shift_op.cc index 6156a2d6af9a010240449a7c944ec0caffc85189..a4150a5664690e750d2501a1849767c23209186b 100644 --- a/paddle/operators/conv_shift_op.cc +++ b/paddle/operators/conv_shift_op.cc @@ -96,14 +96,13 @@ as used in the Neural Turing Machine: https://arxiv.org/abs/1410.5401 The equation is: - \f[ - Out[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} X_{i+j} * Y_{j} - \f] +$$Out[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} X_{i+j} * Y_{j}$$ -where X's index is computed modulo M, and b's index is computed modulo N. +where X's index is computed modulo M, and Y's index is computed modulo N. + +Both inputs X and Y can carry LoD (Level of Details) information. +However, the output only shares the LoD information with input X. -Both of the input `X` and `Y` can carry LoD (Level of Details) information. -However, the output only shares the LoD information with input `X`. )DOC"); } }; diff --git a/paddle/operators/conv_shift_op.cu b/paddle/operators/conv_shift_op.cu index 145e966fe9caa68f7485bb258fa78fd34bfd4c04..74ed1b0ed358afc4f1a4e6a0c322eb032029d551 100644 --- a/paddle/operators/conv_shift_op.cu +++ b/paddle/operators/conv_shift_op.cu @@ -130,9 +130,7 @@ class ConvShiftKernel : public framework::OpKernel { dim3 grid_dim(num_x_blocks, batch_size); - auto stream = reinterpret_cast( - context.device_context()) - .stream(); + auto stream = context.cuda_device_context().stream(); conv_shift_forward<<>>( x_data, y_data, out_data, x_width, y_width, y_half_width, batch_size); @@ -159,9 +157,7 @@ class ConvShiftGradKernel int y_width = Y->dims()[1]; int y_half_width = (y_width - 1) / 2; - auto stream = reinterpret_cast( - context.device_context()) - .stream(); + auto stream = context.cuda_device_context().stream(); const int x_per_block = 256; int num_x_blocks = div_up(x_width, x_per_block); diff --git a/paddle/operators/conv_transpose_op.cc b/paddle/operators/conv_transpose_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..50081779a5ea3c81884007d4e4b7832dc4ea2bdd --- /dev/null +++ b/paddle/operators/conv_transpose_op.cc @@ -0,0 +1,203 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/conv_transpose_op.h" + +namespace paddle { +namespace operators { + +void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(Input) of ConvTransposeOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Filter"), + "Input(Filter) of ConvTransposeOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Output"), + "Output(Output) of ConvTransposeOp should not be null."); + + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + std::vector strides = ctx->Attrs().Get>("strides"); + std::vector paddings = ctx->Attrs().Get>("paddings"); + + for (size_t i = 0; i < paddings.size(); ++i) { + PADDLE_ENFORCE_EQ(paddings[i], 0, + "No Padding allowed in conv transpose op."); + } + + PADDLE_ENFORCE(in_dims.size() == 4 || in_dims.size() == 5, + "ConvTransposeOp intput should be 4-D or 5-D tensor."); + PADDLE_ENFORCE_EQ(in_dims.size(), filter_dims.size(), + "ConvTransposeOp input dimension and filter dimension " + "should be the same."); + PADDLE_ENFORCE(in_dims.size() - strides.size() == 2U, + "ConvTransposeOp input dimension and strides dimension should " + "be consistent."); + PADDLE_ENFORCE_EQ(paddings.size(), strides.size(), + "ConvTransposeOp paddings dimension and Conv strides " + "dimension should be the same."); + PADDLE_ENFORCE_EQ(in_dims[1], filter_dims[0], + "In ConvTransposeOp, The input channel should be the same " + "as the number of filters."); + + std::vector output_shape({in_dims[0], filter_dims[1]}); + for (size_t i = 0; i < paddings.size(); ++i) { + output_shape.push_back((in_dims[i + 2] - 1) * strides[i] + + filter_dims[i + 2]); + } + ctx->SetOutputDim("Output", framework::make_ddim(output_shape)); +} + +Conv2DTransposeOpMaker::Conv2DTransposeOpMaker( + framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "(Tensor) The input tensor of convolution transpose operator. " + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of input channels, H is the height of the feature, and " + "W is the width of the feature."); + AddInput("Filter", + "(Tensor) The filter tensor of convolution transpose operator. " + "The format of the filter tensor is CMHW, where C is the number of " + "output image channels, M is the number of input image channels, " + "H is the height of the filter, and W is the width of the filter. " + "We enforce groups number == 1 and padding == 0 in " + "the convolution transpose scenario."); + AddOutput("Output", + "(Tensor) The output tensor of convolution transpose operator. " + "The format of output tensor is also NCHW."); + AddAttr>( + "strides", + "(vector defalut:{1, 1}), strides of convolution transpose operator.") + .SetDefault({1, 1}); + AddAttr>( + "paddings", + "(vector defalut:{0, 0}), paddings of convolution transpose operator.") + .SetDefault({0, 0}); + AddComment(R"DOC( +Convolution2D Transpose Operator. + +The convolution transpose operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. + +Input(Input, Filter) and output(Output) are in NCHW format. Where N is batch +size, C is the number of channels, H is the height of the feature, and +W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. +These two elements represent height and width, respectively. +The input(X) size and output(Out) size may be different. +Example: + Input: + Input shape: (N, C_in, H_in, W_in) + Filter shape: (C_in, C_out, H_f, W_f) + Output: + Output shape: (N, C_out, H_out, W_out) + where + H_out = (H_in - 1) * strides[0] - 2 * paddings[0] + filter_size[0]; + W_out = (W_in - 1) * strides[1] - 2 * paddings[1] + filter_size[1]; +)DOC"); +} + +Conv3DTransposeOpMaker::Conv3DTransposeOpMaker( + framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Input", + "(Tensor) The input tensor of convolution transpose operator." + "The format of input tensor is NCDHW. Where N is batch size, C is " + "the number of channels, D is the depth of the feature, H is the " + "height of the feature, and " + "W is the width of the feature."); + AddInput("Filter", + "(Tensor) The filter tensor of convolution transpose operator." + "The format of the filter tensor is CMDHW, where C is the number of " + "output image channels, M is the number of input image channels, D " + "is the depth of the filter, H is the height of the filter, and " + "W is the width of the filter." + "We enforce groups number == 1 and padding == 0 in " + "the convolution3d transpose scenario."); + AddOutput("Output", + "(Tensor) The output tensor of convolution transpose operator." + "The format of output tensor is also NCDHW." + "Where N is batch size, C is " + "the number of channels, D is the depth of the feature, H is the " + "height of the feature, and W is the width of the feature."); + AddAttr>( + "strides", + "(vector defalut:{1, 1, 1}), strides of convolution transpose operator.") + .SetDefault({1, 1, 1}); + AddAttr>( + "paddings", + "(vector defalut:{0, 0, 0}), paddings of convolution transpose operator.") + .SetDefault({0, 0, 0}); + AddComment(R"DOC( +Convolution3D Transpose Operator. + +The convolution transpose operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. + +Input(Input, Filter) and output(Output) are in NCDHW format. Where N is batch +size, C is the number of channels, D is the depth of the feature, +H is the height of the feature, and W is the width of the feature. +Parameters(ksize, strides, paddings) are three elements. +These three elements represent depth, height and width, respectively. +The input(X) size and output(Out) size may be different. +Example: + Input: + Input shape: (N, C_in, D_in, H_in, W_in) + Filter shape: (C_in, C_out, D_f, H_f, W_f) + Output: + Output shape: (N, C_out, D_out, H_out, W_out) + where + D_out = (D_in - 1) * strides[0] - 2 * paddings[0] + filter_size[0]; + H_out = (H_in - 1) * strides[1] - 2 * paddings[1] + filter_size[1]; + W_out = (W_in - 1) * strides[2] - 2 * paddings[2] + filter_size[2]; +)DOC"); +} + +void ConvTransposeOpGrad::InferShape(framework::InferShapeContext* ctx) const { + auto in_dims = ctx->GetInputDim("Input"); + auto filter_dims = ctx->GetInputDim("Filter"); + if (ctx->HasOutput(framework::GradVarName("Input"))) { + ctx->SetOutputDim(framework::GradVarName("Input"), in_dims); + } + if (ctx->HasOutput(framework::GradVarName("Filter"))) { + ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims); + } +} + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP(conv2d_transpose, ops::ConvTransposeOp, ops::Conv2DTransposeOpMaker, + conv2d_transpose_grad, ops::ConvTransposeOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv2d_transpose, + ops::GemmConvTransposeKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_transpose_grad, + ops::GemmConvTransposeGradKernel); + +REGISTER_OP(conv3d_transpose, ops::ConvTransposeOp, ops::Conv3DTransposeOpMaker, + conv3d_transpose_grad, ops::ConvTransposeOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv3d_transpose, + ops::GemmConvTransposeKernel); +REGISTER_OP_CPU_KERNEL( + conv3d_transpose_grad, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv2dtranspose_op.cu b/paddle/operators/conv_transpose_op.cu similarity index 59% rename from paddle/operators/conv2dtranspose_op.cu rename to paddle/operators/conv_transpose_op.cu index 761bc1959e69be94f43571728e6b92a322558b99..401cddb379ced134b800d2a078fe130a2850fbb2 100644 --- a/paddle/operators/conv2dtranspose_op.cu +++ b/paddle/operators/conv_transpose_op.cu @@ -12,13 +12,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/conv2dtranspose_op.h" +#include "paddle/operators/conv_transpose_op.h" namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL( - conv2dtranspose, - ops::GemmConv2DTransposeKernel); + conv2d_transpose, + ops::GemmConvTransposeKernel); REGISTER_OP_GPU_KERNEL( - conv2dtranspose_grad, - ops::GemmConv2DTransposeGradKernel); + conv2d_transpose_grad, + ops::GemmConvTransposeGradKernel); + +REGISTER_OP_GPU_KERNEL( + conv3d_transpose, + ops::GemmConvTransposeKernel); +REGISTER_OP_GPU_KERNEL( + conv3d_transpose_grad, + ops::GemmConvTransposeGradKernel); diff --git a/paddle/operators/conv_transpose_op.h b/paddle/operators/conv_transpose_op.h new file mode 100644 index 0000000000000000000000000000000000000000..6c1a6220d784abf89ec789f94d9cff9e5414db04 --- /dev/null +++ b/paddle/operators/conv_transpose_op.h @@ -0,0 +1,293 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/im2col.h" +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/vol2col.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using DDim = framework::DDim; + +// Define Op classes in .h file so that other conv transpose +// operator implementations can reuse the code. +class Conv2DTransposeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Conv2DTransposeOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker); +}; + +class Conv3DTransposeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Conv3DTransposeOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker); +}; + +class ConvTransposeOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override; +}; + +class ConvTransposeOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override; +}; + +template +class GemmConvTransposeKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + // The filter will be reshaped, so it should not be constant pointer + Tensor filter = *context.Input("Filter"); + Tensor* output = context.Output("Output"); + + std::vector strides = context.Attr>("strides"); + // TODO(Zhuoyuan): Paddings can be added in future. + // groups will alway be disabled in conv2dtranspose. + + const int batch_size = static_cast(input->dims()[0]); + + // input_shape_vec: {h, w} or {d, h, w} + std::vector input_shape_vec = framework::vectorize(input->dims()); + input_shape_vec.erase(input_shape_vec.begin(), input_shape_vec.begin() + 2); + + // filter_shape_vec: {k_h, k_w} or {k_d, k_h, k_w} + std::vector filter_shape_vec = framework::vectorize(filter.dims()); + filter_shape_vec.erase(filter_shape_vec.begin(), + filter_shape_vec.begin() + 2); + + // use col_shape in the im2col and col2im (or vol2col and col2vol) + // calculation + // col_shape_vec: {c, k_h, k_w, h, w} or {c, k_d, k_h, k_w, d, h, w} + std::vector col_shape_vec; + col_shape_vec.push_back(output->dims()[1]); + col_shape_vec.insert(col_shape_vec.end(), filter_shape_vec.begin(), + filter_shape_vec.end()); + col_shape_vec.insert(col_shape_vec.end(), input_shape_vec.begin(), + input_shape_vec.end()); + DDim col_shape(framework::make_ddim(col_shape_vec)); + + // use col_matrix_shape in the gemm calculation + // size: (c * k_h * k_w, h * w) or (c * k_d * k_h * k_w, d * h * w) + DDim col_matrix_shape = + framework::flatten_to_2d(col_shape, filter_shape_vec.size() + 1); + + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix; + col_matrix.ShareDataWith(col); + col_matrix.Resize(col_matrix_shape); + + // output size: (c, o_h, o_w) or (c, o_d, o_h, o_w) + DDim output_shape = + framework::slice_ddim(output->dims(), 1, output->dims().size()); + + // input matrix size: (m, h * w) or (m, d * h * w) + DDim input_matrix_shape = {input->dims()[1], col_matrix_shape[1]}; + + // filter size: (m, c * k_h * k_w) or (m, c * k_d * k_h * k_w) + DDim filter_matrix_shape = {input->dims()[1], col_matrix_shape[0]}; + filter.Resize(filter_matrix_shape); + + output->mutable_data(context.GetPlace()); + math::SetConstant set_zero; + set_zero(context.device_context(), output, static_cast(0)); + + // convolution transpose: gemm + col2im or col2vol (similar to conv-backward + // on input) + for (int i = 0; i < batch_size; i++) { + // batch with size (m, h * w) or (m, d * h * w) + Tensor input_batch = input->Slice(i, i + 1).Resize(input_matrix_shape); + + // output size: (c, o_h, o_w) or (c, o_d, o_h, o_w) + Tensor output_batch = output->Slice(i, i + 1).Resize(output_shape); + + // col_matrix = filter * input_batch + // of shape (c * k_h * k_w, h * w) or (c * k_d * k_h * k_w, d * h * w) + math::matmul(context.device_context(), filter, true, + input_batch, false, static_cast(1.0), + &col_matrix, static_cast(0.0)); + + if (filter_shape_vec.size() == 2) { + // col2im: col_matrix -> dy + // from (c * k_h * k_w, h * w) to (c, o_h, o_w) + math::Col2ImFunctor col2im; + + col2im(context.device_context(), output_batch, col, strides[0], + strides[1], 0, 0, 0, 0); + } else if (filter_shape_vec.size() == 3) { + // col2vol: col_matrix -> dy + // from (c * k_d * k_h * k_w, d * h * w) to (c, o_d, o_h, o_w) + math::Col2VolFunctor col2vol; + col2vol(context.device_context(), output_batch, col, strides[0], + strides[1], strides[2], 0, 0, 0); + } + } + } +}; + +template +class GemmConvTransposeGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + const Tensor* output_grad = + context.Input(framework::GradVarName("Output")); + // For filter, we do not use const pointer b/c we will do reshape, + // but we should avoid modifying its value. + Tensor filter = *context.Input("Filter"); + Tensor* input_grad = + context.Output(framework::GradVarName("Input")); + Tensor* filter_grad = + context.Output(framework::GradVarName("Filter")); + + if ((!input_grad) && (!filter_grad)) return; + + std::vector strides = context.Attr>("strides"); + // Actually, no paddings and groups allowed in conv transpose. + std::vector paddings = context.Attr>("paddings"); + + const int batch_size = static_cast(input->dims()[0]); + + // input_shape_vec: {h, w} or {d, h, w} + std::vector input_shape_vec = framework::vectorize(input->dims()); + input_shape_vec.erase(input_shape_vec.begin(), input_shape_vec.begin() + 2); + + // filter_shape_vec: {k_h, k_w} or {k_d, k_h, k_w} + std::vector filter_shape_vec = framework::vectorize(filter.dims()); + filter_shape_vec.erase(filter_shape_vec.begin(), + filter_shape_vec.begin() + 2); + + // use col_shape in the im2col and col2im (or vol2col and col2vol) + // calculation + // col_shape_vec: {c, k_h, k_w, h, w} or {c, k_d, k_h, k_w, d, h, w} + std::vector col_shape_vec; + col_shape_vec.push_back(output_grad->dims()[1]); + col_shape_vec.insert(col_shape_vec.end(), filter_shape_vec.begin(), + filter_shape_vec.end()); + col_shape_vec.insert(col_shape_vec.end(), input_shape_vec.begin(), + input_shape_vec.end()); + DDim col_shape(framework::make_ddim(col_shape_vec)); + + // use col_matrix_shape in the gemm calculation + // size: (c * k_h * k_w, h * w) or (c * k_d * k_h * k_w, d * h * w) + DDim col_matrix_shape = + framework::flatten_to_2d(col_shape, filter_shape_vec.size() + 1); + + // output size: (c, o_h, o_w) or (c, o_d, o_h, o_w) + DDim output_shape = framework::slice_ddim(output_grad->dims(), 1, + output_grad->dims().size()); + + // input matrix size: (m, h * w) or (m, d * h * w) + DDim input_matrix_shape = {input->dims()[1], col_matrix_shape[1]}; + + // filter size: (m, c * k_h * k_w) or (m, c * k_d * k_h * k_w) + DDim filter_matrix_shape = {input->dims()[1], col_matrix_shape[0]}; + filter.Resize(filter_matrix_shape); + + // convolution transpose grad on input: + // im2col + gemm (similar to conv-forward) + // input need to compute gradient + if (input_grad || filter_grad) { + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix; + col_matrix.ShareDataWith(col); + col_matrix.Resize(col_matrix_shape); + + Tensor filter_grad_; + math::SetConstant set_zero; + + if (input_grad) { + input_grad->mutable_data(context.GetPlace()); + set_zero(context.device_context(), input_grad, static_cast(0)); + } + if (filter_grad) { // filter size (m, c, k_h, k_w) + filter_grad->mutable_data(context.GetPlace()); + set_zero(context.device_context(), filter_grad, static_cast(0)); + filter_grad_ = *filter_grad; + filter_grad_.Resize(filter_matrix_shape); + } + + for (int i = 0; i < batch_size; i++) { + // batch with size (c, o_h * o_w) + Tensor output_grad_batch = + output_grad->Slice(i, i + 1).Resize(output_shape); + + if (filter_shape_vec.size() == 2) { + // im2col: dy -> col matrix + // from (c, o_h, o_w) to (c * k_h * k_w, h * w) + math::Im2ColFunctor im2col; + im2col(context.device_context(), output_grad_batch, col, strides[0], + strides[1], paddings[0], paddings[0], paddings[1], + paddings[1]); + } else if (filter_shape_vec.size() == 3) { + // vol2col: dy -> col_matrix + // from (c, o_d, o_h, o_w) to (c * k_d * k_h * k_w, d * h * w) + math::Vol2ColFunctor vol2col; + vol2col(context.device_context(), output_grad_batch, col, strides[0], + strides[1], strides[2], paddings[0], paddings[1], + paddings[2]); + } + + if (input_grad) { + // batch with size (m, h, w) + Tensor input_grad_batch = + input_grad->Slice(i, i + 1).Resize(input_matrix_shape); + // gemm: dx = filter * dy + // (m, c * k_h * k_w) * (c * k_h * k_w, h * w) -> (m, h * w) + // or + // (m, c * k_d * k_h * k_w) * (c * k_d * k_h * k_w, d * h * w) -> (m, + // d, h, w) + math::matmul(context.device_context(), filter, false, + col_matrix, false, static_cast(1.0), + &input_grad_batch, static_cast(0.0)); + } + if (filter_grad) { + // input batch + Tensor in_batch = input->Slice(i, i + 1).Resize(input_matrix_shape); + // gemm: d_filter = x * dy^T + // (m, c * h * w) * (k_h * k_w, c * h * w) -> (m, k_h * k_w) + // or + // (m, d * h * w) * (d * h * w, c * k_d * k_h * k_w) -> (m, c * k_d * + // k_h * k_w) + math::matmul(context.device_context(), in_batch, false, + col_matrix, true, static_cast(1.0), + &filter_grad_, static_cast(1.0)); + } + } + } + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/cos_sim_op.cc b/paddle/operators/cos_sim_op.cc index 55f69fb03ad69c94dc4ebb8edd651d84e06a5f46..312264ccd48d1405a247a2c864d9f5897c897bea 100644 --- a/paddle/operators/cos_sim_op.cc +++ b/paddle/operators/cos_sim_op.cc @@ -79,15 +79,16 @@ class CosSimOpMaker : public framework::OpProtoAndCheckerMaker { AddComment(R"DOC( Cosine Similarity Operator. -The equation is: Out = X^T * Y / (sqrt(X^T * X) * sqrt(Y^T * Y)). +$Out = X^T * Y / (\sqrt{X^T * X} * \sqrt{Y^T * Y})$ -The input `X` and `Y` must have the same shape, except that the 1st dimension -of input `Y` could be just 1 (different from input `X`), which will be -broadcasted to match the shape of input `X` before computing their cosine +The input X and Y must have the same shape, except that the 1st dimension +of input Y could be just 1 (different from input X), which will be +broadcasted to match the shape of input X before computing their cosine similarity. -Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +Both the input X and Y can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD information with input X. + )DOC"); } }; diff --git a/paddle/operators/crf_decoding_op.cc b/paddle/operators/crf_decoding_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..f418f489c0ff471464a23380598e9f4c8da16ca9 --- /dev/null +++ b/paddle/operators/crf_decoding_op.cc @@ -0,0 +1,138 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/crf_decoding_op.h" + +namespace paddle { +namespace operators { +class CRFDecodingOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CRFDecodingOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Emission", + "(LoDTensor, default: LoDTensor). A LoDTensor with shape " + "[N x D] where N is the size of the mini-batch and D is the total " + "tag number. This input is the unscaled emission weight matrix of " + "the linear_chain_crf operator."); + AddInput( + "Transition", + "(Tensor, default: Tensor). A Tensor with shape [(D + 2) x D]. " + "This input is the transition weights learned by the linear_chain_crf " + "operator, denoted as w. The 1st row of w are transition weights for " + "the start mask. The 2nd row of w are transition weights for the end " + "mask. Transition weights between other tags begin from the 3rd row of " + "w. See more details in comments of the linear_chain_crf operator."); + AddInput( + "Label", + "(LoDTensor, LoDTensor). The ground truth with shape " + "[N x 1]. This input is optional. See more details in the operator's " + "comments.") + .AsDispensable(); + AddOutput("ViterbiPath", + "(LoDTensor, LoDTensor). The decoding results. What to " + "return changes depending on whether the Input(Label) (the groud " + "truth) is given. See more details in the operator's comment."); + AddComment(R"DOC( +The crf_decoding operator reads the emission feature weights and the transition +freature weights learned by the linear_chain_crf operator. It implements the +Viterbi algorithm which is a dynamic programming algorithm for finding the most +likely sequence of hidden states, called the Viterbi path, that results in a +sequence of observed tags. + +The output of this operator changes according to whether Input(Label) is given: + +1. Input(Label) is given: + +This happens in training. This operator is used to co-work with the chunk_eval +operator. + +When Input(Label) is given, the crf_decoding operator returns a row vector +with shape [N x 1] whose values are fixed to be 0, indicating an incorrect +prediction, or 1 indicating a tag is correctly predicted. Such an ouput is the +input to chunk_eval operator. + +2. Input(Label) is not given: + +This is the standard decoding process. + +The crf_decoding operator returns a row vecotr with shape [N x 1] whose values +range from 0 to maximum tag number - 1. Each element indicates an index of a +predicted tag. +)DOC"); + } +}; + +class CRFDecodingOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Emission"), + "Input(Emission) should be not null."); + PADDLE_ENFORCE(ctx->HasInput("Transition"), + "Input(Transition) should be not null."); + + PADDLE_ENFORCE(ctx->HasOutput("ViterbiPath"), + "Output(ViterbiPath) should be not null."); + + auto emission_dims = ctx->GetInputDim("Emission"); + PADDLE_ENFORCE_EQ(emission_dims.size(), 2UL, + "The Input(Emission) should be a 2-D tensor."); + PADDLE_ENFORCE(emission_dims[0], "An empty mini-batch is not allowed."); + + auto transition_dims = ctx->GetInputDim("Transition"); + PADDLE_ENFORCE_EQ(transition_dims.size(), 2UL, + "The Input(Transition) should be a 2-D tensor."); + PADDLE_ENFORCE_EQ( + transition_dims[0] - 2, transition_dims[1], + "An invalid dimension for the Input(Transition), which should " + "be a 2-D tensor with shape [(D + 2) x D]."); + PADDLE_ENFORCE_EQ( + emission_dims[1], transition_dims[1], + "The 2nd dimension of the Input(Emission) and the Input(Transition) " + "should be equal to the tag number."); + + if (ctx->HasInput("Label")) { + auto label_dims = ctx->GetInputDim("Label"); + PADDLE_ENFORCE(label_dims.size() == 2UL && label_dims[1] == 1UL, + "The Input(Label) should be a 2-D tensor with the 2nd " + "dimensions fixed to 1."); + PADDLE_ENFORCE_EQ( + emission_dims[0], label_dims[0], + "The height of Input(Emission) and the height of Input(Label) " + "should be the same."); + } + + ctx->ShareLoD("Emission", /*->*/ "ViterbiPath"); + ctx->SetOutputDim("ViterbiPath", {emission_dims[0], 1}); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Emission")->type()), + ctx.device_context()); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(crf_decoding, ops::CRFDecodingOp, + ops::CRFDecodingOpMaker); +REGISTER_OP_CPU_KERNEL( + crf_decoding, ops::CRFDecodingOpKernel, + ops::CRFDecodingOpKernel); diff --git a/paddle/operators/crf_decoding_op.h b/paddle/operators/crf_decoding_op.h new file mode 100644 index 0000000000000000000000000000000000000000..526e0c5dcb2649b35ee28f5153c8472ca7a0af7b --- /dev/null +++ b/paddle/operators/crf_decoding_op.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using framework::LoDTensor; +using framework::LoD; +using framework::Tensor; + +template +class CRFDecodingOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "The crf_decoding operator can only run on CPU."); + + auto* emission_weights = ctx.Input("Emission"); + auto* transition_weights = ctx.Input("Transition"); + auto* label = ctx.Input("Label"); + auto* decoded_path = ctx.Output("ViterbiPath"); + + PADDLE_ENFORCE_EQ(emission_weights->NumLevels(), 1UL, + "The Input(Emission) should be a sequence."); + auto lod = emission_weights->lod(); + PADDLE_ENFORCE(lod.size(), "Input(Emission) must be a sequence."); + const size_t level = 0; + const size_t seq_num = lod[level].size() - 1; + + int* path = decoded_path->mutable_data(platform::CPUPlace()); + math::SetConstant()(ctx.device_context(), + decoded_path, 0); + for (size_t i = 0; i < seq_num; ++i) { + int start_pos = static_cast(lod[level][i]); + int end_pos = static_cast(lod[level][i + 1]); + Tensor decoded_path_one_seq = decoded_path->Slice(start_pos, end_pos); + Decode(emission_weights->Slice(start_pos, end_pos), *transition_weights, + &decoded_path_one_seq); + } + + if (label) { + PADDLE_ENFORCE_EQ(label->NumLevels(), 1UL, + "The Input(Label) should be a sequence."); + const int* label_value = label->data(); + size_t batch_size = emission_weights->dims()[0]; + for (size_t i = 0; i < batch_size; ++i) { + path[i] = label_value[i] == path[i] ? 1 : 0; + } + } + } + + private: + void Decode(const Tensor& emission_weights, const Tensor& transition_weights, + Tensor* decoded_path) const { + auto emission_dims = emission_weights.dims(); + const size_t seq_len = emission_dims[0]; + const size_t tag_num = emission_dims[1]; + + const size_t state_trans_base_idx = 2; + + const T* x = emission_weights.data(); + const T* w = transition_weights.data(); + int* path = decoded_path->data(); + + // alpha is a memo table. An element alpha(k, v) records the score of the + // best sequence of tags from position 1 to position k with v being the end + // tag. + Tensor alpha; + T* alpha_value = alpha.mutable_data(emission_dims, platform::CPUPlace()); + Tensor track; + int* track_value = + track.mutable_data(emission_dims, platform::CPUPlace()); + + for (size_t i = 0; i < tag_num; ++i) alpha_value[i] = w[i] + x[i]; + + for (size_t k = 1; k < seq_len; ++k) { + for (size_t i = 0; i < tag_num; ++i) { + T max_score = -std::numeric_limits::max(); + int max_j = 0; + for (size_t j = 0; j < tag_num; ++j) { + T score = alpha_value[(k - 1) * tag_num + j] + + w[(j + state_trans_base_idx) * tag_num + i]; + if (score > max_score) { + max_score = score; + max_j = j; + } + } + + alpha_value[k * tag_num + i] = max_score + x[k * tag_num + i]; + track_value[k * tag_num + i] = max_j; + } + } + + T max_score = -std::numeric_limits::max(); + int max_i = 0; + for (size_t i = 0; i < tag_num; ++i) { + T score = alpha_value[(seq_len - 1) * tag_num + i] + w[tag_num + i]; + if (score > max_score) { + max_score = score; + max_i = i; + } + } + path[seq_len - 1] = max_i; + for (int k = seq_len - 1; k >= 1; --k) { + path[k - 1] = max_i = track_value[k * tag_num + max_i]; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc index ed78e9e3a3a49b7ff0990b8d13cfe2dae594b722..6752eb8c1c72150b0b1cf5595211ca1d01ef2bf4 100644 --- a/paddle/operators/crop_op.cc +++ b/paddle/operators/crop_op.cc @@ -56,34 +56,35 @@ class CropOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of pad op. " - "The input should be a k-D tensor(k > 0 and k < 7)"); + "The input should be a k-D tensor(k > 0 and k < 7)."); AddInput("Y", - "The input used as reference for cropping" - " with the same dimension as X. ") + "The input used as reference for cropping, " + "which is of the same dimensions as X.") .AsDispensable(); AddOutput("Out", - "The output of crop op " - "with the same dimension as X."); + "The output of crop op, " + "which is of the same dimensions as X."); AddAttr>("offsets", - "A list describing offsets to be cropped." - "The size of offsets list should be as same as " - "dimension size of input X."); + "A list describing offsets to be cropped. " + "The size of offsets list should be the same as " + "the dimension size of input X."); AddAttr>("shape", - "A list describing the shape of output." - "The size of shape list should be as same as " - "dimension size of input X.") + "A list describing the shape of output. " + "The size of shape list should be the same as " + "the dimension size of input X.") .SetDefault(std::vector()); AddComment(R"DOC( Crop Operator. + Crop input into output, as specified by offsets and shape. There are two ways to set shape: -1. referenc input: crop input X as shape as reference input. +1. reference input: crop input X into the same shape as reference input. The dimension of reference input should - be as same as input X. -2. shape list: crop input X by shape described by a list. - The size of shape list should be as same as - dimension size of input X. + be the same as the dimension of input X. +2. shape list: crop input X into the shape described by a list. + The size of shape list should be the same as + the dimension size of input X. The input should be a k-D tensor(k > 0 and k < 7). As an example: @@ -91,20 +92,20 @@ Given: X = [[0, 1, 2, 0, 0] [0, 3, 4, 0, 0] - [0, 0, 0, 0, 0]] + [0, 0, 0, 0, 0]], and - offsets = [0, 1] + offsets = [0, 1], and - shape = [2, 2] + shape = [2, 2], -then we get +we get: Out = [[1, 2], - [3, 4]] + [3, 4]]. )DOC"); } diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index 39df19da677a7dee7d0989d491f8d5511f73a9c7..1e82742eaf86711fe4f9d02d517ad1853131cf67 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -49,11 +49,13 @@ class CrossEntropyOp : public framework::OperatorWithKernel { } protected: - // Explicitly set that data type of the output of the cross_entropy operator + // Explicitly set that the data type of computation kernel of cross_entropy // is determined by its input "X". - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("X")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); } }; @@ -96,10 +98,13 @@ class CrossEntropyGradientOp : public framework::OperatorWithKernel { } protected: - // CrossEntropy's data type just determined by "X" - framework::DataType IndicateDataType( + // Explicitly set that the data type of computation kernel of cross_entropy + // is determined by its input "X". + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("X")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); } }; @@ -113,21 +118,17 @@ class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { "where N is the batch size and D is the number of classes. " "This input is a probability computed by the previous operator, " "which is almost always the result of a softmax operator."); - AddInput( - "Label", - "(Tensor, default Tensor), the ground truth which is " - "a 2-D tensor. " - "When soft_label is set to false, `Label` is a Tensor with shape " - "[N x 1]. " - "When soft_label is set to true, `Label` is a Tensor " - "with shape [N x K]."); + AddInput("Label", + "(Tensor), the ground truth which is a 2-D tensor. When " + "soft_label is set to false, Label is a Tensor with shape " + "[N x 1]. When soft_label is set to true, Label is a " + "Tensor with shape [N x K]."); AddOutput("Y", - "(Tensor, default Tensor), a 2-D tensor " - "with shape [N x 1]. The cross entropy loss."); - AddAttr( - "soft_label", - "(bool, default false), a flag to indicate whether to interpretate " - "the given labels as soft labels.") + "(Tensor, default Tensor), a 2-D tensor with shape " + "[N x 1]. The cross entropy loss."); + AddAttr("soft_label", + "(bool, default false), a flag indicating whether to " + "interpretate the given labels as soft labels.") .SetDefault(false); AddComment(R"DOC( CrossEntropy Operator. @@ -137,13 +138,13 @@ computation. 1) One-hot cross-entropy: soft_label = false, Label[i, 0] indicates the class index for sample i: - Y[i] = -log(X[i, Label[i]]) + $Y[i] = -\log(X[i, Label[i]])$ 2) Soft-label cross-entropy: soft_label = true, Label[i, j] indicates the soft label of class j for sample i: - Y[i] = \sum_j{-Label[i, j] * log(X[i, j])} + $Y[i] = \sum_j{-Label[i, j] * log(X[i, j])}$ Please make sure that in this case the summuation of each row of Label equals one. @@ -153,8 +154,9 @@ computation. non-zero element (equals 1), soft-label cross-entropy degenerates to a one-hot cross-entropy with one-hot label representation. -Both the input `X` and `Label` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +Both the input X and Label can carry the LoD (Level of Details) information, +or not. But the output only shares the LoD information with input X. + )DOC"); } }; diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu index a523cb6fcec16d309f6bb3baf8549bf14756fd7d..530b319a44eac915f0d49eb55bfe5929908eab26 100644 --- a/paddle/operators/cross_entropy_op.cu +++ b/paddle/operators/cross_entropy_op.cu @@ -82,24 +82,19 @@ class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { int block = 512; int grid = (batch_size * class_num + block - 1) / block; + auto stream = ctx.cuda_device_context().stream(); if (ctx.Attr("soft_label")) { auto* label_data = label->data(); - SoftCrossEntropyGradientKernel<<< - grid, block, 0, reinterpret_cast( - ctx.device_context()) - .stream()>>>(dx_data, dy_data, x_data, label_data, - batch_size, class_num); + SoftCrossEntropyGradientKernel<<>>( + dx_data, dy_data, x_data, label_data, batch_size, class_num); } else { math::SetConstant functor; functor(ctx.device_context(), dx, 0); auto* label_data = label->data(); grid = (batch_size + block - 1) / block; - CrossEntropyGradientKernel<<< - grid, block, 0, reinterpret_cast( - ctx.device_context()) - .stream()>>>(dx_data, dy_data, x_data, label_data, - batch_size, class_num); + CrossEntropyGradientKernel<<>>( + dx_data, dy_data, x_data, label_data, batch_size, class_num); } } }; diff --git a/paddle/operators/decayed_adagrad_op.cc b/paddle/operators/decayed_adagrad_op.cc index 17b394aa07cb0c7ca6e085b61590ff052221b22c..640b4e77448d1b64bcf7375f26c07ff1d2bdeaa3 100644 --- a/paddle/operators/decayed_adagrad_op.cc +++ b/paddle/operators/decayed_adagrad_op.cc @@ -75,11 +75,18 @@ class DecayedAdagradOpMaker : public framework::OpProtoAndCheckerMaker { "Constant for numerical stability") .SetDefault(1.0e-6f); AddComment(R"DOC( +Decayed Adagrad Optimizer. -Decayed Adagrad +The update is done as follows: -moment_out = decay * moment + (1 - decay) * grad * grad -param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon) +$$ +moment\_out = decay * moment + (1 - decay) * grad * grad \\ +param\_out = param - \frac{learning\_rate * grad}{\sqrt{moment\_out} + epsilon} +$$ + +The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) +does not have an epsilon attribute. It is added here for numerical +stability to avoid the division by zero error. )DOC"); } diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc index ff1ccea3b94dcd55c372b707c2afeda874ed212e..818146aca766cb13b93fd024c11c1209655d9e11 100644 --- a/paddle/operators/dropout_op.cc +++ b/paddle/operators/dropout_op.cc @@ -43,22 +43,24 @@ class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { DropoutOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddAttr("dropout_prob", "Probability of setting units to zero.") - .SetDefault(.5f); - AddAttr("is_training", "Whether in training phase.").SetDefault(true); - AddAttr("seed", "Dropout random seed.").SetDefault(0); AddInput("X", "The input of dropout op."); AddOutput("Out", "The output of dropout op."); AddOutput("Mask", "The random sampled dropout mask.").AsIntermediate(); + AddAttr("dropout_prob", "Probability of setting units to zero.") + .SetDefault(.5f); + AddAttr("is_training", "True if in training phase.").SetDefault(true); + AddAttr("seed", "Dropout random seed.").SetDefault(0); + AddComment(R"DOC( Dropout Operator. -'Dropout' refers to randomly dropping out units in a nerual network. It is a +Dropout refers to randomly dropping out units in a nerual network. It is a regularization technique for reducing overfitting by preventing neuron co-adaption during training. The dropout operator randomly set (according to the given dropout probability) the outputs of some units to zero, while others -being set to their inputs. +are set equal to their corresponding inputs. + )DOC"); } }; diff --git a/paddle/operators/dynamic_recurrent_op.cc b/paddle/operators/dynamic_recurrent_op.cc index a0b06ac1dc305bc899f9abaafcc980a6150ecda9..d48cc4e8df587708ab93e7d788145adc01c1d3e5 100644 --- a/paddle/operators/dynamic_recurrent_op.cc +++ b/paddle/operators/dynamic_recurrent_op.cc @@ -386,12 +386,13 @@ class DynamicRecurrentOpProtoAndCheckerMaker RNNAlgorithm::kArgNames[RNNAlgorithm::ComputeMode::kForward]; // inputs and outputs stored in proto AddInput(name.inlinks, - "the inputs that need to be segmented for each step.") + "The inputs that need to be segmented for each step.") .AsDuplicable(); - AddInput(name.initial_states, "variables to initialize states.") + AddInput(name.initial_states, "Variables to initialize the states.") .AsDuplicable(); - AddOutput(name.outlinks, "the outputs that need to concated for all steps.") + AddOutput(name.outlinks, + "The outputs that need to be concatenated for all steps.") .AsDuplicable(); AddOutput(name.step_scopes, "step scopes"); @@ -399,7 +400,12 @@ class DynamicRecurrentOpProtoAndCheckerMaker AddAttr>(name.ex_states, "names of ex_states"); AddAttr>(name.states, "names of states"); - AddComment("This is a RNN operator for varience-length sequences."); + AddComment(R"DOC( +Dynamic Recurrent Operator. + +This is a RNN operator for varience-length sequences. + +)DOC"); } }; diff --git a/paddle/operators/elementwise_add_op.cc b/paddle/operators/elementwise_add_op.cc index d9bc80c869c023caebf0b45ed24f2def3f0b1dd8..ebe1de90c7d245756de759d8675a30f955843798 100644 --- a/paddle/operators/elementwise_add_op.cc +++ b/paddle/operators/elementwise_add_op.cc @@ -22,7 +22,7 @@ class ElementwiseAddOpMaker : public ElementwiseOpMaker { ElementwiseAddOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("add", "Out = X + Y"); + SetComment("Add", "$Out = X + Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_div_op.cc b/paddle/operators/elementwise_div_op.cc index 3f56344d0007b5f14fd9b5b9b44a9b29d3c42f2a..de75816a249002549940b04d928c88c17d075917 100644 --- a/paddle/operators/elementwise_div_op.cc +++ b/paddle/operators/elementwise_div_op.cc @@ -22,7 +22,7 @@ class ElementwiseDivOpMaker : public ElementwiseOpMaker { ElementwiseDivOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Div", "Out = X / Y"); + SetComment("Div", "$Out = X / Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_mul_op.cc b/paddle/operators/elementwise_mul_op.cc index da7765aa6a7a81c9e0b4f462022cad54c16aec47..ffa10486f123963274aa478eb4c607e32138bcec 100644 --- a/paddle/operators/elementwise_mul_op.cc +++ b/paddle/operators/elementwise_mul_op.cc @@ -23,7 +23,7 @@ class ElementwiseMulOpMaker : public ElementwiseOpMaker { ElementwiseMulOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Mul", "Out = X ⊙ Y"); + SetComment("Mul", "$Out = X \\odot\\ Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/elementwise_op.h b/paddle/operators/elementwise_op.h index fce4b24a22f40c9cc57738273a758d0d48ff5e91..56e5eb69bc382a2c15d88b759fa6987f02c6cabb 100644 --- a/paddle/operators/elementwise_op.h +++ b/paddle/operators/elementwise_op.h @@ -46,37 +46,42 @@ class ElementwiseOpMaker : public framework::OpProtoAndCheckerMaker { ElementwiseOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", R"DOC( -The first input of elementwise op, it's a tensor of any dimensions. -)DOC"); - AddInput("Y", R"DOC( -The sencond input of elementwise op, it's a tensor and it's dimensions -must be small or equal to X's dimensions. -)DOC"); + AddInput("X", "(Tensor) The first input tensor of elementwise op"); + AddInput("Y", "(Tensor) The second input tensor of elementwise op"); + AddOutput("Out", "The output of elementwise op"); AddAttr("axis", - R"DOC( -When the shape(Y) does not equal the shape(X),Y will be broadcasted -to match the shape of X and axis should be dimension index Y in X - )DOC") + "(int, default -1) The starting dimension index " + "for broadcasting Y onto X") .SetDefault(-1) .EqualGreaterThan(-1); - - AddOutput("Out", "The output of elementwise op"); comment_ = R"DOC( -Limited elementwise {name} operator.The equation is: Out = {equation}. -1. The shape of Y should be same with X or -2. Y's shape is a subset of X. - Y will be broadcasted to match the shape of X and axis should be dimension index Y in X. - - example: - shape(X) = (2, 3, 4, 5), shape(Y) = (,) - shape(X) = (2, 3, 4, 5), shape(Y) = (5,) - shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) - shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 - shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 +Limited Elementwise {name} Operator. + +The equation is: + +{equation} + +X is a tensor of any dimension and the dimensions of tensor Y must be smaller than +or equal to the dimensions of X. + +There are two cases for this operator: +1. The shape of Y is same with X; +2. The shape of Y is a subset of X. + +For case 2: +Y will be broadcasted to match the shape of X and axis should be +the starting dimension index for broadcasting Y onto X. + +example: + shape(X) = (2, 3, 4, 5), shape(Y) = (,) + shape(X) = (2, 3, 4, 5), shape(Y) = (5,) + shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) + shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 + shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 Both the input X and Y can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input X. +or not. But the output only shares the LoD information with input X. + )DOC"; AddComment(comment_); } diff --git a/paddle/operators/elementwise_sub_op.cc b/paddle/operators/elementwise_sub_op.cc index 3e4f98fdb35b148931a67d511fe41958eb523f99..39702dad0ee61de71ff0d54765e6f73de93cee9c 100644 --- a/paddle/operators/elementwise_sub_op.cc +++ b/paddle/operators/elementwise_sub_op.cc @@ -22,7 +22,7 @@ class ElementwiseSubOpMaker : public ElementwiseOpMaker { ElementwiseSubOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : ElementwiseOpMaker(proto, op_checker) { - SetComment("Sub", "Out = X - Y"); + SetComment("Sub", "$Out = X - Y$"); AddComment(comment_); } }; diff --git a/paddle/operators/expand_op.cc b/paddle/operators/expand_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..282775fcda45fe3bbd72bf04a7ae828f2c840ab7 --- /dev/null +++ b/paddle/operators/expand_op.cc @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/expand_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class ExpandOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null."); + + std::vector expand_times = + ctx->Attrs().Get>("expand_times"); + auto x_dims = ctx->GetInputDim("X"); + + PADDLE_ENFORCE_EQ(static_cast(x_dims.size()), expand_times.size(), + "The number of Attr(expand_times)'s value must be equal " + "to the rank of Input(X)."); + PADDLE_ENFORCE_LE(x_dims.size(), 6, + "The rank of Input(X) must not be greater than 6."); + + std::vector out_shape(x_dims.size()); + for (size_t i = 0; i < expand_times.size(); ++i) { + PADDLE_ENFORCE_GE(expand_times[i], 1, + "Each value of Attr(expand_times) should not be " + "less than 1."); + out_shape[i] = x_dims[i] * expand_times[i]; + } + + ctx->SetOutputDim("Out", framework::make_ddim(out_shape)); + if (out_shape[0] == x_dims[0]) { + ctx->ShareLoD("X", "Out"); + } + } +}; + +class ExpandOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ExpandOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor, default Tensor) A tensor with rank in [1, 6]." + "X is the input tensor to be expanded."); + AddOutput("Out", + "(Tensor, default Tensor) A tensor with rank in [1, 6]." + "The rank of Output(Out) is same as Input(X) except that each " + "dimension size of Output(Out) is equal to corresponding " + "dimension size of Input(X) multiplying corresponding value of " + "Attr(expand_times)."); + AddAttr>("expand_times", + "Expand times number for each dimension."); + AddComment(R"DOC( +Expand operator tiles the input by given times number. You should set times +number for each dimension by providing attribute 'expand_times'. The rank of X +should be in [1, 6]. Please notice that size of 'expand_times' must be same with +X's rank. Following is a using case: + +Input(X) is a 3-D tensor with shape [2, 3, 1]: + + [ + [[1], [2], [3]], + [[4], [5], [6]] + ] + +Attr(expand_times): [1, 2, 2] + +Output(Out) is a 3-D tensor with shape [2, 6, 2]: + + [ + [[1, 1], [2, 2], [3, 3], [1, 1], [2, 2], [3, 3]], + [[4, 4], [5, 5], [6, 6], [4, 4], [5, 5], [6, 6]] + ] + +)DOC"); + } +}; + +class ExpandGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null."); + + auto x_dims = ctx->GetInputDim("X"); + std::vector expand_times = + ctx->Attrs().Get>("expand_times"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + + for (size_t i = 0; i < expand_times.size(); ++i) { + PADDLE_ENFORCE_EQ(x_dims[i] * expand_times[i], out_dims[i], + "Each dimension size of Input(Out@GRAD) should be " + "equal to multiplication of crroresponding dimension " + "size of Input(X) and Attr(expand_times) value."); + } + + auto x_grad_name = framework::GradVarName("X"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(expand, ops::ExpandOp, ops::ExpandOpMaker, expand_grad, + ops::ExpandGradOp); +REGISTER_OP_CPU_KERNEL(expand, + ops::ExpandKernel); +REGISTER_OP_CPU_KERNEL( + expand_grad, ops::ExpandGradKernel); diff --git a/paddle/operators/fill_constant_op.cu b/paddle/operators/expand_op.cu similarity index 69% rename from paddle/operators/fill_constant_op.cu rename to paddle/operators/expand_op.cu index a57b11c6cba77ad7d258c47a8ebf887f359f9522..6744562b6c21dd8bfeb7e4cb6b809dc7913aa3a5 100644 --- a/paddle/operators/fill_constant_op.cu +++ b/paddle/operators/expand_op.cu @@ -13,11 +13,11 @@ limitations under the License. */ #define EIGEN_USE_GPU -#include "paddle/framework/op_registry.h" -#include "paddle/operators/fill_constant_op.h" + +#include "paddle/operators/expand_op.h" namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(expand, + ops::ExpandKernel); REGISTER_OP_GPU_KERNEL( - fill_constant, ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel); + expand_grad, ops::ExpandGradKernel); diff --git a/paddle/operators/expand_op.h b/paddle/operators/expand_op.h new file mode 100644 index 0000000000000000000000000000000000000000..8ae2c11a5d31dafc1b90d129054ebfabfb761bfe --- /dev/null +++ b/paddle/operators/expand_op.h @@ -0,0 +1,172 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +#define MAX_RANK_SUPPORTED 6 + +#define EXPAND_TEMPLATE(z, n, data) \ + case n + 1: { \ + Expand(context); \ + break; \ + } +#define REP_EXPAND_TEMPLATE(n) BOOST_PP_REPEAT(n, EXPAND_TEMPLATE, ~) +#define COND(n) \ + BOOST_PP_GREATER_EQUAL(BOOST_PP_DIV(n, MAX_RANK_SUPPORTED), \ + BOOST_PP_MOD(n, MAX_RANK_SUPPORTED)) +#define EXPAND_GRAD_CASE(n) \ + case n: { \ + ExpandBackward(context, reshape_dims_vec, reduce_dims_vec); \ + break; \ + } +#define EXPAND_GRAD_TEMPLATE(z, n, data) \ + BOOST_PP_IF(COND(n), EXPAND_GRAD_CASE(n), ) +#define REP_EXPAND_GRAD_TEMPLATE(n) BOOST_PP_REPEAT(n, EXPAND_GRAD_TEMPLATE, ~) + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; +template +using EigenTensor = framework::EigenTensor; + +template +class ExpandKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto rank = context.Input("X")->dims().size(); + switch (rank) { + REP_EXPAND_TEMPLATE(MAX_RANK_SUPPORTED) + default: + PADDLE_ENFORCE(false, + "Only support tensor with rank being between 1 and 6."); + } + } + + protected: + template + void Expand(const framework::ExecutionContext& context) const { + auto* in0 = context.Input("X"); + auto& expand_times = context.Attr>("expand_times"); + auto* out0 = context.Output("Out"); + Eigen::DSizes bcast_dims; + auto x_dims = in0->dims(); + for (size_t i = 0; i < expand_times.size(); ++i) { + bcast_dims[i] = expand_times[i]; + } + auto x = EigenTensor::From(*in0); + out0->mutable_data(context.GetPlace()); + auto y = EigenTensor::From(*out0); + auto place = context.GetEigenDevice(); + y.device(place) = x.broadcast(bcast_dims); + } +}; + +template +class ExpandGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto& expand_times = context.Attr>("expand_times"); + auto x_dims = in0->dims(); + // 1. reshape_dims_vec is the broadcast parameter. For each dimension i, + // if expand_times[i] > 1 and x_dims[i] > 1, i will be splitted to two + // dimensions [expand_times[i], x_dims[i]]. + // 2. reduce_dims_vec is the dimension parameter to compute gradients. For + // each dimension expanded, the gradients should be summed to original + // size. + std::vector reshape_dims_vec; + std::vector reduce_dims_vec; + for (size_t i = 0; i < expand_times.size(); ++i) { + if (expand_times[i] == 1) { + reshape_dims_vec.push_back(x_dims[i]); + } else { + if (x_dims[i] == 1) { + reduce_dims_vec.push_back(reshape_dims_vec.size()); + reshape_dims_vec.push_back(expand_times[i]); + } else { + reduce_dims_vec.push_back(reshape_dims_vec.size()); + reshape_dims_vec.push_back(expand_times[i]); + reshape_dims_vec.push_back(x_dims[i]); + } + } + } + + int dims = reshape_dims_vec.size() * MAX_RANK_SUPPORTED + + reduce_dims_vec.size() - MAX_RANK_SUPPORTED - 1; + // no need reduce, just copy + if (reduce_dims_vec.size() == 0) { + auto* in0 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + out0->mutable_data(context.GetPlace()); + out0->CopyFrom(*in0, context.GetPlace(), context.device_context()); + } else { + switch (dims) { + REP_EXPAND_GRAD_TEMPLATE(72) + default: + PADDLE_ENFORCE( + false, "Only support tensor with rank being between 1 and 6."); + } + } + } + + protected: + template + void ExpandBackward(const framework::ExecutionContext& context, + const std::vector& reshape_dims_vec, + const std::vector& reduce_dims_vec) const { + size_t reshape_size = Dims / MAX_RANK_SUPPORTED + 1; + size_t reduce_size = Dims % MAX_RANK_SUPPORTED + 1; + PADDLE_ENFORCE_EQ(reshape_size, reshape_dims_vec.size(), + "Inconsistent size between template Dims and " + "reshape dimensions."); + PADDLE_ENFORCE_EQ(reduce_size, reduce_dims_vec.size(), + "Inconsistent size between template Dims and " + "reduce dimensions."); + auto* in0 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + auto x = EigenVector::Flatten(*(context.Input("X"))); + out0->mutable_data(context.GetPlace()); + auto x_grad = EigenVector::Flatten(*out0); + Eigen::DSizes reshape_dims; + for (size_t i = 0; i < reshape_size; ++i) { + reshape_dims[i] = reshape_dims_vec[i]; + } + Eigen::DSizes reduce_dims; + for (size_t i = 0; i < reduce_size; ++i) { + reduce_dims[i] = reduce_dims_vec[i]; + } + auto out_grad = EigenVector::Flatten(*in0); + x_grad.device(context.GetEigenDevice()) = + out_grad.reshape(reshape_dims).sum(reduce_dims).reshape(x.dimensions()); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/feed_op.cc b/paddle/operators/feed_op.cc index 0e5b263eae904d97b61d41691b848e4fa2c17971..0dd84cbeaafbafd45132b0a0b744554ce7475411 100644 --- a/paddle/operators/feed_op.cc +++ b/paddle/operators/feed_op.cc @@ -59,8 +59,13 @@ class FeedOpInfoMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of feed op"); AddOutput("Out", "The output of feed op"); - AddComment("feed op, it should not be configured by users directly"); - AddAttr("col", "column of feed"); + AddAttr("col", "(int) The column of feed"); + AddComment(R"DOC( +Feed Operator. + +It should not be configured by users directly. + +)DOC"); } }; diff --git a/paddle/operators/fetch_op.cc b/paddle/operators/fetch_op.cc index f1086e3dc774a5e57f1abb5d4f91f859fc0e64aa..8108ae69dec4bafd1c04d5ab05eef6f467d4c6e8 100644 --- a/paddle/operators/fetch_op.cc +++ b/paddle/operators/fetch_op.cc @@ -66,8 +66,13 @@ class FetchOpInfoMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of fetch op"); AddOutput("Out", "The output of fetch op"); - AddComment("fetch op, it should not be configured by users directly"); - AddAttr("col", "column of fetch"); + AddAttr("col", "(int) The column of fetch"); + AddComment(R"DOC( +Fetch Operator. + +It should not be configured by users directly. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/fill_constant_batch_size_like_op.cc b/paddle/operators/fill_constant_batch_size_like_op.cc index 58c9f1cd2c79c150aaed7753641f6ad6120dd0f5..85871ebbfcd8ee38ef5e8078d1d6cb6bdda46a7b 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cc +++ b/paddle/operators/fill_constant_batch_size_like_op.cc @@ -34,16 +34,26 @@ class FillConstantBatchSizeLikeOp : public framework::OperatorWithKernel { std::vector shape_int64(shape.size(), 0); std::transform(shape.begin(), shape.end(), shape_int64.begin(), [](int a) { return static_cast(a); }); - auto dims = framework::make_ddim(shape_int64); + auto output_dim = framework::make_ddim(shape_int64); - dims[0] = ctx->GetInputDim("Input")[0]; - ctx->SetOutputDim("Out", dims); + int input_dim_idx = ctx->Attrs().Get("input_dim_idx"); + PADDLE_ENFORCE_GE(input_dim_idx, 0); + PADDLE_ENFORCE_GT(ctx->GetInputDim("Input").size(), input_dim_idx); + + int output_dim_idx = ctx->Attrs().Get("output_dim_idx"); + PADDLE_ENFORCE_GE(output_dim_idx, 0); + PADDLE_ENFORCE_GT(static_cast(shape.size()), output_dim_idx); + + output_dim[output_dim_idx] = ctx->GetInputDim("Input")[input_dim_idx]; + ctx->SetOutputDim("Out", output_dim); } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext &ctx) const override { - return static_cast(ctx.Attr("data_type")); + return framework::OpKernelType( + static_cast(ctx.Attr("data_type")), + ctx.device_context()); } }; @@ -57,25 +67,37 @@ class FillConstantBatchSizeLikeOpMaker "(int, default 5 (FP32)) " "Output data type") .SetDefault(framework::DataType::FP32); - AddAttr>("shape", "(vector) The shape of the output"); - AddAttr("value", "(float, default 0) The value to be filled") - .SetDefault(0.0f); AddInput("Input", "(Tensor) Tensor " - "whose first dimension is used to specify the batch_size"); + "whose dim_idx th dimension is used to specify the batch_size"); AddOutput("Out", "(Tensor) Tensor of specified shape will be filled " "with the specified value"); - AddComment(R"DOC(Fill up a variable with specified constant value.)DOC"); + AddAttr>("shape", "(vector) The shape of the output"); + AddAttr("input_dim_idx", + "(int, default 0) The index of input's batch size dimension") + .SetDefault(0); + AddAttr("output_dim_idx", + "(int, default 0) The index of output's batch size dimension") + .SetDefault(0); + AddAttr("value", "(float, default 0) The value to be filled") + .SetDefault(0.0f); + AddComment(R"DOC( +FillConstantBatchSizeLike Operator. + +Fill up a variable with specified constant value. + +)DOC"); } }; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(fill_constant_batch_size_like, - ops::FillConstantBatchSizeLikeOp, - ops::FillConstantBatchSizeLikeOpMaker); +REGISTER_OPERATOR(fill_constant_batch_size_like, + ops::FillConstantBatchSizeLikeOp, + paddle::framework::EmptyGradOpMaker, + ops::FillConstantBatchSizeLikeOpMaker); REGISTER_OP_CPU_KERNEL( fill_constant_batch_size_like, ops::FillConstantBatchSizeLikeOpKernel, diff --git a/paddle/operators/fill_constant_batch_size_like_op.cu b/paddle/operators/fill_constant_batch_size_like_op.cu index cfa5df001e9d6c606751e3ca3cddda02812ef180..298c196f1dfef388640e34153264986bd518a11a 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.cu +++ b/paddle/operators/fill_constant_batch_size_like_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/framework/op_registry.h" #include "paddle/operators/fill_constant_batch_size_like_op.h" diff --git a/paddle/operators/fill_constant_batch_size_like_op.h b/paddle/operators/fill_constant_batch_size_like_op.h index a360e6683ec7204ea5bdbe27ca88a0ac51c983ac..339d97a30a5819ab488e83990651ba99212239ec 100644 --- a/paddle/operators/fill_constant_batch_size_like_op.h +++ b/paddle/operators/fill_constant_batch_size_like_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -27,9 +27,8 @@ class FillConstantBatchSizeLikeOpKernel : public framework::OpKernel { out->mutable_data(ctx.GetPlace()); auto value = ctx.Attr("value"); - auto out_eigen = framework::EigenVector::Flatten(*out); - auto place = ctx.GetEigenDevice(); - out_eigen.device(place) = out_eigen.constant(static_cast(value)); + math::SetConstant setter; + setter(ctx.device_context(), out, static_cast(value)); } }; diff --git a/paddle/operators/fill_constant_op.cc b/paddle/operators/fill_constant_op.cc index 7a861b6cfc0fab312f4e5a7cce2fc28f923173d2..818f113b90a4c239a857791fb9957e51d3287b97 100644 --- a/paddle/operators/fill_constant_op.cc +++ b/paddle/operators/fill_constant_op.cc @@ -12,30 +12,41 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/fill_constant_op.h" +#include "paddle/framework/data_type.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { -class FillConstantOp : public framework::OperatorWithKernel { +class FillConstantInferShape : public framework::InferShapeBase { public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext *ctx) const override { + void operator()(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of FillConstantOp should not be null."); auto &shape = ctx->Attrs().Get>("shape"); - std::vector shape_int64(shape.size(), 0); - std::transform(shape.begin(), shape.end(), shape_int64.begin(), - [](int a) { return static_cast(a); }); - auto dims = framework::make_ddim(shape_int64); - ctx->SetOutputDim("Out", dims); + ctx->SetOutputDim("Out", framework::make_ddim(shape)); } +}; - protected: - framework::DataType IndicateDataType( - const framework::ExecutionContext &ctx) const override { - return static_cast(ctx.Attr("data_type")); +class FillConstantOp : public framework::OperatorBase { + public: + using framework::OperatorBase::OperatorBase; + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto data_type = static_cast(Attr("data_type")); + auto value = Attr("value"); + auto force_cpu = Attr("force_cpu"); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + out.Resize(framework::make_ddim(Attr>("shape"))); + if (force_cpu) { + auto cpu = platform::CPUPlace(); + out.mutable_data(cpu, framework::ToTypeIndex(data_type)); + } else { + out.mutable_data(dev_ctx.GetPlace(), framework::ToTypeIndex(data_type)); + } + math::set_constant(dev_ctx, &out, value); } }; @@ -51,19 +62,26 @@ class FillConstantOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr>("shape", "(vector) The shape of the output"); AddAttr("value", "(float, default 0) The value to be filled") .SetDefault(0.0f); + AddAttr("force_cpu", + "(bool, default false) Force fill output variable to cpu " + "memory. Otherwise, fill output variable to the running " + "device") + .SetDefault(false); AddOutput("Out", "(Tensor) Tensor of specified shape will be filled " "with the specified value"); - AddComment(R"DOC(Fill up a variable with specified constant value.)DOC"); + AddComment(R"DOC( +FillConstantBatchSizeLike Operator. + +Fill up a variable with specified constant value. + +)DOC"); } }; } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_WITHOUT_GRADIENT(fill_constant, ops::FillConstantOp, - ops::FillConstantOpMaker); -REGISTER_OP_CPU_KERNEL( - fill_constant, ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel); +REGISTER_OPERATOR(fill_constant, ops::FillConstantOp, + ops::FillConstantInferShape, ops::FillConstantOpMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index ed529ac40aaf179b35a9ab32e11ed7dbbe9289ba..8ab39d4fb012b8fa3883f33e4d15be7918500354 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -37,11 +37,13 @@ class FillZerosLikeOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker *op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of fill-zeros-like op."); - AddOutput("Y", "The varibale will be filled up with zeros."); + AddOutput("Y", "The variable will be filled up with zeros."); AddComment(R"DOC( -Fill up a vriable with zeros. +FillZerosLike Operator. + +Fill up a variable with zeros. +The output will have the same size as the input. -The output will have the same size with input. )DOC"); } }; diff --git a/paddle/operators/fill_zeros_like_op.cu b/paddle/operators/fill_zeros_like_op.cu index fdbcf520a0d7b4ddfe3fc1837a21e0ce88b8e8fa..a6d4ba64bde534ea76867c456537b130a45b9496 100644 --- a/paddle/operators/fill_zeros_like_op.cu +++ b/paddle/operators/fill_zeros_like_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/framework/op_registry.h" #include "paddle/operators/fill_zeros_like_op.h" diff --git a/paddle/operators/fill_zeros_like_op.h b/paddle/operators/fill_zeros_like_op.h index cdf56a723b117fe7b08ef2749aa2c2978c923d44..7e7d78eea2bce427d6ad4dfb77bcb4ace35cd287 100644 --- a/paddle/operators/fill_zeros_like_op.h +++ b/paddle/operators/fill_zeros_like_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" namespace paddle { namespace operators { @@ -23,10 +23,11 @@ template class FillZerosLikeKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* output = context.Output("Y"); - output->mutable_data(context.GetPlace()); - auto t = framework::EigenVector::Flatten(*output); - t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + auto* out = context.Output("Y"); + out->mutable_data(context.GetPlace()); + + math::SetConstant setter; + setter(context.device_context(), out, static_cast(0)); } }; diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc index f6c7f472da24a1a60c0d2538ae643bdc8e55b10f..8f80fb162519f60fcce897b3c31a3507bbf6ba6d 100644 --- a/paddle/operators/gather_op.cc +++ b/paddle/operators/gather_op.cc @@ -40,9 +40,11 @@ class GatherOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("X")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); } }; @@ -55,9 +57,11 @@ class GatherGradOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("X")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); } }; @@ -67,11 +71,28 @@ class GatherOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The source input of gather op"); AddInput("Index", "The index input of gather op"); - AddOutput("Out", "The output of add op"); + AddOutput("Out", "The output of gather op"); AddComment(R"DOC( -Gather Operator by selecting from the first axis, +Gather Operator. + +$Out = X[Index]$ + +Out is obtained by gathering entries of the outer-most dimension +of X indexed by Index and concatenate them together. + +Example: + +X = [[1, 2], + [3, 4], + [5, 6]] + +Index = [[1, 2]] + +Then: + +Out = [[3, 4], + [5, 6]] -Out = X[Index] )DOC"); } }; diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index be7f542a7a274d88d2dac953995d6a83a6ce022d..53ad86c6c48d1868f4495af51661d91b39a84f0b 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -57,9 +57,11 @@ class GaussianRandomOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return static_cast(ctx.Attr("data_type")); + return framework::OpKernelType( + static_cast(ctx.Attr("data_type")), + ctx.device_context()); } }; @@ -68,21 +70,35 @@ class GaussianRandomOpMaker : public framework::OpProtoAndCheckerMaker { GaussianRandomOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "output matrix of random op"); - AddComment(R"DOC( -GaussianRandom operator. -Use to initialize tensor with gaussian random generator. -)DOC"); + AddOutput("Out", "Output matrix of gaussian random op"); - AddAttr>("shape", "The dimension of random tensor."); - AddAttr("mean", "mean of random tensor.").SetDefault(.0f); - AddAttr("std", "std of random tensor.").SetDefault(1.0f); + AddAttr>("shape", + "(vector) " + "The dimension of random tensor."); + AddAttr("mean", + "(float, default 0.0) " + "mean of random tensor.") + .SetDefault(.0f); + AddAttr("std", + "(float, default 1.0) " + "std of random tensor.") + .SetDefault(1.0f); AddAttr("seed", + "(int, default 0) " "Random seed of generator." - "0 means use system wide seed") + "0 means use system wide seed.") .SetDefault(0); - AddAttr("data_type", "output data type") + AddAttr("data_type", + "(int, default 5(FP32)) " + "Output data type.") .SetDefault(framework::DataType::FP32); + + AddComment(R"DOC( +GaussianRandom Operator. + +Used to initialize tensors with gaussian random generator. + +)DOC"); } }; diff --git a/paddle/operators/gru_op.cc b/paddle/operators/gru_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..5aa03f8916a67222fb0ca5781533766063e52683 --- /dev/null +++ b/paddle/operators/gru_op.cc @@ -0,0 +1,220 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/gru_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class GRUOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(%s) of GRUOp should not be null.", "Input"); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(%s) of GRUOp should not be null.", "Weight"); + PADDLE_ENFORCE(ctx->HasOutput("BatchGate"), + "Output(%s) of GRUOp should not be null.", "BatchGate"); + PADDLE_ENFORCE(ctx->HasOutput("BatchResetHiddenPrev"), + "Output(%s) of GRUOp should not be null.", + "BatchResetHiddenPrev"); + PADDLE_ENFORCE(ctx->HasOutput("BatchHidden"), + "Output(%s) of GRUOp should not be null.", "BatchHidden"); + PADDLE_ENFORCE(ctx->HasOutput("Hidden"), + "Output(%s) of GRUOp should not be null.", "Hidden"); + auto input_dims = ctx->GetInputDim("Input"); + auto weight_dims = ctx->GetInputDim("Weight"); + int input_size = input_dims[1]; + int frame_size = weight_dims[0]; + PADDLE_ENFORCE_EQ(input_size, frame_size * 3, + "The input_size must be 3 times of frame_size in GRUOp."); + PADDLE_ENFORCE_EQ( + weight_dims[1], frame_size * 3, + "The shape of Weight matrix must be [frame_size, frame_size * 3]."); + if (ctx->HasInput("H0")) { + auto h0_dims = ctx->GetInputDim("H0"); + PADDLE_ENFORCE_EQ(h0_dims[1], frame_size, + "The width of H0 must be equal to frame_size."); + } + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + int bias_height = bias_dims[0]; + int bias_width = bias_dims[1]; + PADDLE_ENFORCE_EQ(bias_height, 1, + "The shape of Bias must be [1, frame_size * 3]."); + PADDLE_ENFORCE_EQ(bias_width, frame_size * 3, + "The shape of Bias must be [1, frame_size * 3]."); + } + ctx->SetOutputDim("BatchGate", input_dims); + ctx->SetOutputDim("BatchResetHiddenPrev", {input_dims[0], frame_size}); + ctx->SetOutputDim("BatchHidden", {input_dims[0], frame_size}); + ctx->SetOutputDim("Hidden", {input_dims[0], frame_size}); + ctx->ShareLoD("Input", "Hidden"); + } +}; + +class GRUOpMaker : public framework::OpProtoAndCheckerMaker { + public: + GRUOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Input", + "(LoDTensor) The first input is a LodTensor, which supports " + "variable-time length input sequence. The underlying tensor in " + "this LoDTenosr is a matrix with shape (T X 3D), where, T is the " + "total time steps in this mini-batch, D is the hidden size."); + AddInput("H0", + "(Tensor, optional) The initial hidden state is an optional " + "input. This is a tensor with shape (N x D), where N is the " + "batch size, D is the hidden size.") + .AsDispensable(); + AddInput( + "Weight", + "(Tensor) The learnable hidden-hidden weight matrix with shape " + "(D x 3D), where D is the hidden size. The elements continuous in " + "memory can be divided into two parts. The first part are weights of " + "the update gate and reset gate with shape (D x 2D), and the second " + "part are weights of output candidate with shape (D x D)."); + AddInput("Bias", + "(Tensor, optional) Bias vector with shape (1 x 3D) concating " + "bias of the update gate, reset gate and output candidate.") + .AsDispensable(); + AddOutput("BatchGate", + "(LoDTensor) To compute with batches, sequence data will be " + "reorganized into several successive batches each containing " + "data from the same time step. The LoDTensor BatchGate contains " + "the update gate, reset gate and output candidate values " + "organized in batches. The LoD size is 2. The first LoD contains " + "the batch offsets and the second LoD contains the indexes in " + "the raw sequence data.") + .AsIntermediate(); + AddOutput( + "BatchResetHiddenPrev", + "(LoDTensor) The reseted hidden state LoDTensor organized in batches. " + "This LoDTensor is a matrix with shape (T X D) and has the same LoD " + "with `BatchGate`.") + .AsIntermediate(); + AddOutput( + "BatchHidden", + "(LoDTensor) The hidden state LoDTensor organized in batches. " + "This LoDTensor is a matrix with shape (T X D) and has the same LoD " + "with `BatchGate`.") + .AsIntermediate(); + AddOutput( + "Hidden", + "(LoDTensor) the hidden state LoDTensor organized in sequences. " + "This LoDTensor is a matrix with shape (T X D) and has the same LoD " + "with `BatchGate`."); + AddAttr("activation", + "(string, default tanh) " + "The activation type used for output candidate {h}_t.") + .SetDefault("tanh"); + AddAttr( + "gate_activation", + "(string, default sigmoid) " + "The activation type used in update gate and reset gate.") + .SetDefault("sigmoid"); + AddAttr("is_reverse", + "(bool, defalut: False) " + "whether to compute reversed GRU.") + .SetDefault(false); + AddComment(R"DOC( +GRU Operator implements part calculations of the complete GRU as following: + +\f[ +update \ gate: u_t = actGate(xu_t + W_u * h_{t-1} + b_u) \\ +reset \ gate: r_t = actGate(xr_t + W_r * h_{t-1} + b_r) \\ +output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, h_{t-1}) + b_c) \\ +output: h_t = dot((1 - u_t), h_{t-1}) + dot(u_t, {h}_t) +\f] + +@note To implement the complete GRU, fully-connected operator must be used +before to feed xu, xr and xc as the Input of GRU operator. +)DOC"); + } +}; + +class GRUGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Input"), + "Input(%s) of GRUGradOp should not be null.", "Input"); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(%s) of GRUGradOp should not be null.", "Weight"); + PADDLE_ENFORCE(ctx->HasInput("BatchGate"), + "Input(%s) of GRUGradOp should not be null.", "BatchGate"); + PADDLE_ENFORCE(ctx->HasInput("BatchResetHiddenPrev"), + "Input(%s) of GRUGradOp should not be null.", + "BatchResetHiddenPrev"); + PADDLE_ENFORCE(ctx->HasInput("BatchHidden"), + "Input(%s) of GRUOp should not be null.", "BatchHidden"); + PADDLE_ENFORCE(ctx->HasInput("Hidden"), + "Input(%s) of GRUGradOp should not be null.", "Hidden"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Hidden")), + "Input(%s@GRAD) of GRUGradOp should not be null.", "Hidden"); + auto input_dims = ctx->GetInputDim("Input"); + auto weight_dims = ctx->GetInputDim("Weight"); + int input_size = input_dims[1]; + int frame_size = weight_dims[0]; + int weight_height = weight_dims[0]; + int weight_width = weight_dims[1]; + PADDLE_ENFORCE_EQ(input_size, frame_size * 3, + "The input_size must be 3 times of frame_size in GRUOp."); + PADDLE_ENFORCE_EQ( + weight_height, frame_size, + "The shape of Weight matrix must be [frame_size, frame_size * 3]."); + PADDLE_ENFORCE_EQ( + weight_width, frame_size * 3, + "The shape of Weight matrix must be [frame_size, frame_size * 3]."); + if (ctx->HasInput("H0")) { + auto h0_dims = ctx->GetInputDim("H0"); + PADDLE_ENFORCE_EQ(h0_dims[1], frame_size, + "The width of H0 must be equal to frame_size."); + auto h0_grad_name = framework::GradVarName("H0"); + if (ctx->HasOutput(h0_grad_name)) + ctx->SetOutputDim(h0_grad_name, h0_dims); + } + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + int bias_height = bias_dims[0]; + int bias_width = bias_dims[1]; + PADDLE_ENFORCE_EQ(bias_height, 1, + "The shape of Bias must be [1, frame_size * 3]."); + PADDLE_ENFORCE_EQ(bias_width, frame_size * 3, + "The shape of Bias must be [1, frame_size * 3]."); + auto bias_grad_name = framework::GradVarName("Bias"); + if (ctx->HasOutput(bias_grad_name)) + ctx->SetOutputDim(bias_grad_name, bias_dims); + } + auto input_grad_name = framework::GradVarName("Input"); + if (ctx->HasOutput(input_grad_name)) + ctx->SetOutputDim(input_grad_name, input_dims); + auto weight_grad_name = framework::GradVarName("Weight"); + if (ctx->HasOutput(weight_grad_name)) + ctx->SetOutputDim(weight_grad_name, weight_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(gru, ops::GRUOp, ops::GRUOpMaker, gru_grad, ops::GRUGradOp); +REGISTER_OP_CPU_KERNEL(gru, ops::GRUKernel, + ops::GRUKernel); +REGISTER_OP_CPU_KERNEL(gru_grad, + ops::GRUGradKernel, + ops::GRUGradKernel); diff --git a/paddle/operators/gru_op.cu b/paddle/operators/gru_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..35538c74b4bf678f8068999bfadb2589a1671be0 --- /dev/null +++ b/paddle/operators/gru_op.cu @@ -0,0 +1,23 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/gru_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(gru, ops::GRUKernel, + ops::GRUKernel); +REGISTER_OP_GPU_KERNEL(gru_grad, + ops::GRUGradKernel, + ops::GRUGradKernel); diff --git a/paddle/operators/gru_op.h b/paddle/operators/gru_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ba90ec9816c40a6a49065ac6efcee6b93dffce90 --- /dev/null +++ b/paddle/operators/gru_op.h @@ -0,0 +1,231 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/operators/math/gru_compute.h" +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/sequence2batch.h" + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; + +template +using EigenMatrix = framework::EigenMatrix; + +template +class GRUKernel : public framework::OpKernel { + public: + void BatchCompute(const framework::ExecutionContext& context) const { + auto* input = context.Input("Input"); + auto* h0 = context.Input("H0"); + const T* h0_data = h0 ? h0->data() : nullptr; + auto* weight = context.Input("Weight"); + const T* weight_data = weight->data(); + auto* bias = context.Input("Bias"); + auto* batch_gate = context.Output("BatchGate"); + batch_gate->mutable_data(context.GetPlace()); + auto* batch_reset_hidden_prev = + context.Output("BatchResetHiddenPrev"); + batch_reset_hidden_prev->mutable_data(context.GetPlace()); + auto* batch_hidden = context.Output("BatchHidden"); + batch_hidden->mutable_data(context.GetPlace()); + auto* hidden = context.Output("Hidden"); + hidden->mutable_data(context.GetPlace()); + + context.ShareLoD("Input", "Hidden"); + + auto hidden_dims = hidden->dims(); + + bool is_reverse = context.Attr("is_reverse"); + math::LoDTensor2BatchFunctor to_batch; + to_batch(context.device_context(), *input, *batch_gate, true, is_reverse); + + int frame_size = hidden_dims[1]; + int batch_size = hidden_dims[0]; + auto g = EigenMatrix::From(*batch_gate); + auto place = context.GetEigenDevice(); + if (bias) { + auto b = EigenMatrix::From(*bias); + g.device(place) = g + + b.reshape(Eigen::array({{1, frame_size * 3}})) + .broadcast(Eigen::array({{batch_size, 1}})); + } + + math::hl_gru_value gru_value; + gru_value.gateWeight = const_cast(weight_data); + gru_value.stateWeight = + const_cast(weight_data + 2 * frame_size * frame_size); + gru_value.prevOutValue = const_cast(h0_data); + auto batch_starts = batch_gate->lod()[0]; + size_t num_batch = batch_starts.size() - 1; + for (size_t n = 0; n < num_batch; n++) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + int cur_batch_size = bend - bstart; + + Tensor gate_t = batch_gate->Slice(bstart, bend); + Tensor reset_hidden_prev_t = batch_reset_hidden_prev->Slice(bstart, bend); + Tensor hidden_t = batch_hidden->Slice(bstart, bend); + gru_value.outputValue = hidden_t.data(); + gru_value.gateValue = gate_t.data(); + gru_value.resetOutputValue = reset_hidden_prev_t.data(); + math::GRUUnitFunctor::compute( + context.device_context(), gru_value, frame_size, cur_batch_size, + math::ActiveType(context.Attr("activation")), + math::ActiveType(context.Attr("gate_activation"))); + gru_value.prevOutValue = gru_value.outputValue; + } + + math::Batch2LoDTensorFunctor to_seq; + batch_hidden->set_lod(batch_gate->lod()); + to_seq(context.device_context(), *batch_hidden, *hidden); + } + + void Compute(const framework::ExecutionContext& context) const override { + BatchCompute(context); + } +}; + +template +class GRUGradKernel : public framework::OpKernel { + public: + void BatchCompute(const framework::ExecutionContext& context) const { + auto* h0 = context.Input("H0"); + const T* h0_data = h0 ? h0->data() : nullptr; + auto* weight = context.Input("Weight"); + const T* weight_data = weight->data(); + auto* batch_gate = context.Input("BatchGate"); + auto* batch_reset_hidden_prev = + context.Input("BatchResetHiddenPrev"); + auto* batch_hidden = context.Input("BatchHidden"); + auto* hidden = context.Input("Hidden"); + auto* hidden_grad = + context.Input(framework::GradVarName("Hidden")); + auto* input_grad = + context.Output(framework::GradVarName("Input")); + auto* h0_grad = context.Output(framework::GradVarName("H0")); + auto* weight_grad = + context.Output(framework::GradVarName("Weight")); + auto* bias_grad = context.Output(framework::GradVarName("Bias")); + + auto gate_dims = batch_gate->dims(); + auto hidden_dims = hidden->dims(); + int frame_size = hidden_dims[1]; + + math::LoDTensor2BatchFunctor to_batch; + LoDTensor batch_hidden_grad, batch_gate_grad, batch_reset_hidden_prev_grad; + batch_hidden_grad.mutable_data(hidden_dims, context.GetPlace()); + batch_gate_grad.mutable_data(gate_dims, context.GetPlace()); + batch_reset_hidden_prev_grad.mutable_data(hidden_dims, + context.GetPlace()); + math::SetConstant zero; + zero(context.device_context(), &batch_hidden_grad, static_cast(0.0)); + zero(context.device_context(), &batch_gate_grad, static_cast(0.0)); + zero(context.device_context(), &batch_reset_hidden_prev_grad, + static_cast(0.0)); + + bool is_reverse = context.Attr("is_reverse"); + batch_hidden_grad.set_lod(batch_hidden->lod()); + to_batch(context.device_context(), *hidden_grad, batch_hidden_grad, false, + is_reverse); + + math::hl_gru_value gru_value; + gru_value.gateWeight = const_cast(weight_data); + gru_value.stateWeight = + const_cast(weight_data + 2 * frame_size * frame_size); + + math::hl_gru_grad gru_grad; + if (weight_grad) { + gru_grad.gateWeightGrad = + weight_grad->mutable_data(context.GetPlace()); + zero(context.device_context(), weight_grad, static_cast(0.0)); + gru_grad.stateWeightGrad = + weight_grad->data() + 2 * frame_size * frame_size; + } else { + gru_grad.gateWeightGrad = nullptr; + gru_grad.stateWeightGrad = nullptr; + } + + auto batch_starts = batch_hidden_grad.lod()[0]; + size_t num_batch = batch_starts.size() - 1; + for (int n = static_cast(num_batch) - 1; n >= 0; n--) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + int cur_batch_size = bend - bstart; + + Tensor gate_t = batch_gate->Slice(bstart, bend); + gru_value.gateValue = gate_t.data(); + Tensor reset_hidden_prev_t = batch_reset_hidden_prev->Slice(bstart, bend); + gru_value.resetOutputValue = reset_hidden_prev_t.data(); + + Tensor hidden_grad_t = batch_hidden_grad.Slice(bstart, bend); + gru_grad.outputGrad = hidden_grad_t.data(); + Tensor gate_grad_t = batch_gate_grad.Slice(bstart, bend); + gru_grad.gateGrad = gate_grad_t.data(); + Tensor reset_hidden_prev_grad_t = + batch_reset_hidden_prev_grad.Slice(bstart, bend); + gru_grad.resetOutputGrad = reset_hidden_prev_grad_t.data(); + if (n == 0) { + gru_value.prevOutValue = const_cast(h0_data); + if (h0_grad) { + T* h0_grad_data = h0_grad->mutable_data(context.GetPlace()); + zero(context.device_context(), h0_grad, static_cast(0.0)); + gru_grad.prevOutGrad = h0_grad_data; + } else { + gru_grad.prevOutGrad = nullptr; + } + } else { + int bstart_pre = static_cast(batch_starts[n - 1]); + Tensor hidden_prev_t = batch_hidden->Slice(bstart_pre, bstart); + gru_value.prevOutValue = hidden_prev_t.data(); + Tensor hidden_prev_grad_t = batch_hidden_grad.Slice(bstart_pre, bstart); + gru_grad.prevOutGrad = hidden_prev_grad_t.data(); + } + + math::GRUUnitGradFunctor::compute( + context.device_context(), gru_value, gru_grad, frame_size, + cur_batch_size, + math::ActiveType(context.Attr("activation")), + math::ActiveType(context.Attr("gate_activation"))); + } + if (input_grad) { + input_grad->mutable_data(context.GetPlace()); + math::Batch2LoDTensorFunctor to_seq; + batch_gate_grad.set_lod(batch_gate->lod()); + to_seq(context.device_context(), batch_gate_grad, *input_grad); + } + if (bias_grad) { + bias_grad->mutable_data(context.GetPlace()); + auto d_b = EigenMatrix::From(*bias_grad); + auto d_g = EigenMatrix::From(batch_gate_grad); + auto place = context.GetEigenDevice(); + d_b.device(place) = d_g.sum(Eigen::array({{0}})); + } + } + + void Compute(const framework::ExecutionContext& context) const override { + BatchCompute(context); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/gru_unit_op.cc b/paddle/operators/gru_unit_op.cc index 8d9723289d9af9ef218a5e056b4b585383e00dac..89c027ff1eea93012dc5ab22b081786efc328e96 100644 --- a/paddle/operators/gru_unit_op.cc +++ b/paddle/operators/gru_unit_op.cc @@ -80,19 +80,21 @@ class GRUUnitOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("HiddenPrev", "(Tensor) Matrix with shape [batch_size, frame_size] for the " "states of previous time step."); - AddInput("Weight", - "(Tensor) Weight matrix with shape [frame_size, frame_size * 3]. " - "The elements continuous in memory can be divided into two parts. " - "The first part are weights of the update gate and reset gate " - "with shape [frame_size, frame_size * 2], and the second part are " - "weights of output candidate with shape [frame_size, frame_size]"); - AddInput("Bias", - "(Tensor) Bias vector with shape [1, frame_size * 3] concating " - "bias of the update gate, reset gate and output candidate.") + AddInput( + "Weight", + "(Tensor) Weight matrix with shape [frame_size, frame_size * 3]. " + "The elements continuous in memory can be divided into two parts. " + "The first part are weights of the update gate and reset gate " + "with shape [frame_size, frame_size * 2], and the second part are " + "weights of output candidate with shape [frame_size, frame_size]."); + AddInput( + "Bias", + "(Tensor) Bias vector with shape [1, frame_size * 3] concatenating " + "bias of the update gate, reset gate and output candidate.") .AsDispensable(); AddOutput("Gate", "(Tensor) Matrix with shape [batch_size, frame_size * 3] for the " - "output of update gate, reset gate and output candidate") + "output of update gate, reset gate and output candidate.") .AsIntermediate(); AddOutput("ResetHiddenPrev", "(Tensor) Matrix with shape [batch_size, frame_size] for the " @@ -112,16 +114,19 @@ class GRUUnitOpMaker : public framework::OpProtoAndCheckerMaker { .SetDefault(sigmoid) .InEnum({identity, sigmoid, tanh, relu}); AddComment(R"DOC( -GRUUnitOp implements part calculations of the GRU unit as following: +GRUUnit Operator. -\f[ -update \ gate: u_t = actGate(xu_t + W_u * hidden_prev + bias_u) \\ -reset \ gate: r_t = actGate(xr_t + W_r * hidden_prev + bias_r) \\ -output \ candidate: {h}_t = actNode(xc_t + W_c * dot(r_t, hidden_prev) + bias_c) \\ -output: h_t = dot((1-u_t), {h}_t) + dot(u_t, hidden_prev) -\f] +This operator implements partial calculations of the GRU unit as follows: + +$$ +update \ gate: u_t = actGate(xu_t + W_u * hidden_{prev} + bias_u) \\ +reset \ gate: r_t = actGate(xr_t + W_r * hidden_{prev} + bias_r) \\ +output \ candidate: {h}_t = actNode({xc}_t + W_c * dot(r_t, hidden_{prev}) + bias_c) \\ +output: h_t = dot((1-u_t), {h}_t) + dot(u_t, hidden_{prev}) +$$ The rest of GRU unit can be completed by using FCOp's output as the input of GRUUnitOp. + )DOC"); } }; diff --git a/paddle/operators/huber_loss_op.cc b/paddle/operators/huber_loss_op.cc index 2d9449f5ca50dab8d2a7928c4311ec2d66b47904..3435e74b0afb470fcbd1c0f4e06ad363352cac00 100644 --- a/paddle/operators/huber_loss_op.cc +++ b/paddle/operators/huber_loss_op.cc @@ -59,10 +59,12 @@ class HuberLossOpMaker : public framework::OpProtoAndCheckerMaker { "The shape is same as Input(X) and will be reused in backward.") .AsIntermediate(); AddOutput("Out", - "The output tensor with shape [batch_size, 1] which represents " - "the huber loss."); + "The output tensor with shape [batch_size, 1] " + "which represents the huber loss."); AddAttr("delta", "Hyper parameter in huber loss."); AddComment(R"DOC( +HuberLoss Operator. + Huber loss is a loss function used in robust regression. We define X as the input value and Y as the target value. Huber loss can evaluate the fitness of X to Y. Different from MSE loss, Huber loss is more robust for outliers. The diff --git a/paddle/operators/increment_op.cc b/paddle/operators/increment_op.cc index 139392c691e00b2a94f46801f1cfc2018ce139f5..35efb12932f1d61fdb511b4ee2cdab3891507c61 100644 --- a/paddle/operators/increment_op.cc +++ b/paddle/operators/increment_op.cc @@ -12,26 +12,60 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/increment_op.h" +#include "paddle/framework/op_registry.h" namespace paddle { namespace operators { -class IncrementOp : public framework::OperatorWithKernel { +class IncrementInferShape : public framework::InferShapeBase { public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext *ctx) const override { + void operator()(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of IncrementOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of IncrementOp should not be null."); + PADDLE_ENFORCE_EQ(1, framework::product(ctx->GetInputDim("X"))); ctx->SetOutputDim("Out", ctx->GetInputDim("X")); - ctx->ShareLoD("X", /*->*/ "Out"); } }; -template +struct IncrementFunctor { + IncrementFunctor(const framework::LoDTensor &x, framework::LoDTensor *out, + float value) + : x_(x), out_(out), value_(value) {} + + template + void operator()() const { + *out_->data() = *x_.data() + static_cast(value_); + } + + const framework::LoDTensor &x_; + framework::LoDTensor *out_; + float value_; +}; + +class IncrementOp : public framework::OperatorBase { + public: + IncrementOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + + PADDLE_ENFORCE(platform::is_cpu_place(x.place())); + out.Resize(x.dims()); + out.mutable_data(x.place(), x.type()); + float value = Attr("step"); + framework::VisitDataType(framework::ToDataType(out.type()), + IncrementFunctor(x, &out, value)); + } +}; + class IncrementOpMaker : public framework::OpProtoAndCheckerMaker { public: IncrementOpMaker(framework::OpProto *proto, @@ -39,14 +73,18 @@ class IncrementOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) The input tensor of increment operator"); AddOutput("Out", "(Tensor) The output tensor of increment operator."); - AddComment(R"DOC(Increment operator + AddAttr("step", + "(float, default 1.0) " + "The step size by which the " + "input tensor will be incremented.") + .SetDefault(1.0); + AddComment(R"DOC( +Increment Operator. + +The equation is: +$$Out = X + step$$ -The equation is: Out = X + step )DOC"); - AddAttr("step", - "The step size by which the " - "input tensor will be incremented.") - .SetDefault(1.0); } }; @@ -56,10 +94,10 @@ class IncrementGradOpMaker : public framework::SingleGradOpDescMaker { std::unique_ptr Apply() const override { auto *grad_op = new framework::OpDescBind(); - grad_op->SetType("scale"); - grad_op->SetInput("X", OutputGrad("Out")); - grad_op->SetOutput("Out", InputGrad("X")); - grad_op->SetAttr("scale", 1.0f); + grad_op->SetType("increment"); + grad_op->SetInput("X", Output("Out")); + grad_op->SetOutput("Out", Input("X")); + grad_op->SetAttr("step", -boost::get(GetAttr("step"))); return std::unique_ptr(grad_op); } }; @@ -68,8 +106,5 @@ class IncrementGradOpMaker : public framework::SingleGradOpDescMaker { } // namespace paddle namespace ops = paddle::operators; - -REGISTER_OPERATOR(increment, ops::IncrementOp, ops::IncrementOpMaker, - ops::IncrementGradOpMaker); -REGISTER_OP_CPU_KERNEL(increment, - ops::IncrementKernel); +REGISTER_OPERATOR(increment, ops::IncrementOp, ops::IncrementInferShape, + ops::IncrementOpMaker, ops::IncrementGradOpMaker); diff --git a/paddle/operators/increment_op.h b/paddle/operators/increment_op.h deleted file mode 100644 index 342e254fc453555c70923efbca02fdfd014af015..0000000000000000000000000000000000000000 --- a/paddle/operators/increment_op.h +++ /dev/null @@ -1,40 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once - -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { -template -class IncrementKernel : public framework::OpKernel { - public: - virtual void Compute(const framework::ExecutionContext& context) const { - auto* tensor = context.Output("Out"); - auto* in = context.Input("X"); - tensor->mutable_data(in->place()); - - auto step = static_cast(context.Attr("step")); - - auto eigen_out = framework::EigenVector::Flatten(*tensor); - auto eigen_in = framework::EigenVector::Flatten(*in); - auto& place = context.GetEigenDevice(); - eigen_out.device(place) = eigen_in + step; - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/l1_norm_op.cc b/paddle/operators/l1_norm_op.cc index 1d111696cf43d232413a8dec7ffb057cb1913c7f..02ebf022968e95d0b20598d3c935fb51177c8841 100644 --- a/paddle/operators/l1_norm_op.cc +++ b/paddle/operators/l1_norm_op.cc @@ -57,7 +57,7 @@ L1 Norm Operator. Computes the L1 norm of a tensor. -Out = sum (abs(X)) +$$Out = \sum{|X|}$$ )DOC"); } diff --git a/paddle/operators/linear_chain_crf_op.cc b/paddle/operators/linear_chain_crf_op.cc index 605dbba5af1bb8b0d718833be6af45fdaeac70ac..066bdf67aa037e9c25cfdfaff7ec8771eb59cde8 100644 --- a/paddle/operators/linear_chain_crf_op.cc +++ b/paddle/operators/linear_chain_crf_op.cc @@ -22,52 +22,55 @@ class LinearChainCRFOpMaker : public framework::OpProtoAndCheckerMaker { LinearChainCRFOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "Emission", - "(LoDTensor, default: LoDTensor). " - "The unscaled emission weight matrix for the linear chain CRF. " - "This input is a LoDTensor with shape [N x D] where N is the size of " - "the mini-batch and D is the total tag number."); - AddInput( - "Transition", - "(Tensor, default: Tensor). A Tensor with shape [(D + 2) x D]. " - "The learnable parameter for the linear_chain_crf operator. " - "See more details in the operator's comments."); - AddInput( - "Label", - "(LoDTensor, default: LoDTensor). The ground truth which is a 2-D " - "LoDTensor with shape [N x 1], where N is the total element number in " - "a mini-batch."); + AddInput("Emission", + "(LoDTensor, default LoDTensor) " + "A 2-D LoDTensor with shape [N x D], where N is the size of the " + "mini-batch and D is the total tag number. The unscaled emission " + "weight matrix for the linear chain CRF. "); + AddInput("Transition", + "(Tensor, default Tensor) A 2-D Tensor with shape " + "[(D + 2) x D]. The learnable parameter for the linear_chain_crf " + "operator. See more details in the operator's comments."); + AddInput("Label", + "(LoDTensor, default LoDTensor) A LoDTensor with shape " + "[N x 1], where N is the total element number in a mini-batch. " + "The ground truth."); AddOutput( "Alpha", - "Tensor, default: Tensor. The forward vectors for the entire " - "batch. A two dimensional tensor with shape [N x D], " - "denoted as \f$\alpha\f$. \f$\alpha$\f is a memo table used to " - "calculate the normalization factor in CRF. \f$\alpha[k, v]$\f stores " - "the unnormalized probabilites of all possible unfinished sequences of " - "tags that end at position \f$k$\f with tag \f$v$\f. For each \f$k$\f, " + "(Tensor, default Tensor) A 2-D Tensor with shape [N x D]. " + "The forward vectors for the entire batch. Denote it as \f$\alpha\f$. " + "\f$\alpha$\f is a memo table used to calculate the normalization " + "factor in CRF. \f$\alpha[k, v]$\f stores the unnormalized " + "probabilites of all possible unfinished sequences of tags that end at " + "position \f$k$\f with tag \f$v$\f. For each \f$k$\f, " "\f$\alpha[k, v]$\f is a vector of length \f$D$\f with a component for " "each tag value \f$v$\f. This vector is called a forward vecotr and " "will also be used in backward computations.") .AsIntermediate(); - AddOutput("EmissionExps", - "The exponentials of Input(Emission). This is an intermediate " - "computational result in forward computation, and will be reused " - "in backward computation.") + AddOutput( + "EmissionExps", + "(Tensor, default Tensor) A 2-D Tensor with shape [N x D]. " + "The exponentials of Input(Emission). This is an intermediate " + "computational result in forward computation, and will be reused in " + "backward computation.") .AsIntermediate(); - AddOutput("TransitionExps", - "The exponentials of Input(Transition). This is an intermediate " - "computational result in forward computation, and will be reused " - "in backward computation.") + AddOutput( + "TransitionExps", + "(Tensor, default Tensor) A 2-D Tensor with shape " + "[(D + 2) x D]. The exponentials of Input(Transition). This is an " + "intermediate computational result in forward computation, and " + "will be reused in backward computation.") .AsIntermediate(); AddOutput( "LogLikelihood", - "(Tensor, default: Tensor). The logarithm of the conditional " + "(Tensor, default Tensor) The logarithm of the conditional " "likelihood of each training sample in a mini-batch. This is a 2-D " "tensor with shape [S x 1], where S is the sequence number in a " "mini-batch. Note: S is equal to the sequence number in a mini-batch. " "The output is no longer a LoDTensor."); AddComment(R"DOC( +LinearChainCRF Operator. + Conditional Random Field defines an undirected probabilistic graph with nodes denoting random variables and edges denoting dependencies between these variables. CRF learns the conditional probability \f$P(Y|X)\f$, where @@ -81,29 +84,28 @@ and output must be linear sequences. Thus, the graph of such a CRF is a simple chain or a line, which results in the linear chain CRF. This operator implements the Forward-Backward algorithm for the linear chain -CRF. Please see http://www.cs.columbia.edu/~mcollins/fb.pdf and -http://cseweb.ucsd.edu/~elkan/250Bwinter2012/loglinearCRFs.pdf for reference. +CRF. Please refer to http://www.cs.columbia.edu/~mcollins/fb.pdf and +http://cseweb.ucsd.edu/~elkan/250Bwinter2012/loglinearCRFs.pdf for details. Equation: - -- Denote Input(Emission) to this operator as \f$x\f$ here. -- The first D values of Input(Transition) to this operator are for starting +1. Denote Input(Emission) to this operator as \f$x\f$ here. +2. The first D values of Input(Transition) to this operator are for starting weights, denoted as \f$a\f$ here. -- The next D values of Input(Transition) of this operator are for ending +3. The next D values of Input(Transition) of this operator are for ending weights, denoted as \f$b\f$ here. -- The remaning values of Input(Transition) are for transition weights, +4. The remaning values of Input(Transition) are for transition weights, denoted as \f$w\f$ here. -- Denote Input(Label) as \f$s\f$ here. +5. Denote Input(Label) as \f$s\f$ here. The probability of a sequence \f$s\f$ of length \f$L\f$ is defined as: -\f$P(s) = (1/Z) exp(a_{s_1} + b_{s_L} +\f$P(s) = (1/Z) \exp(a_{s_1} + b_{s_L} + \sum_{l=1}^L x_{s_l} + \sum_{l=2}^L w_{s_{l-1},s_l})\f$ where \f$Z\f$ is a normalization value so that the sum of \f$P(s)\f$ over all possible sequences is \f$1\f$, and \f$x\f$ is the emission feature weight to the linear chain CRF. -Finaly, the linear chain CRF operator outputs the logarithm of the conditional +Finally, the linear chain CRF operator outputs the logarithm of the conditional likelihood of each training sample in a mini-batch. NOTE: @@ -179,11 +181,13 @@ class LinearChainCRFOp : public framework::OperatorWithKernel { } protected: - // Explicitly set that the data type of output of the linear_chain_crf - // operator is determined by its input "Emission". - framework::DataType IndicateDataType( + // Explicitly set that the data type of computation kernel of linear_chain_crf + // is determined by its input "Emission". + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("Emission")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Emission")->type()), + ctx.device_context()); } }; @@ -238,10 +242,13 @@ class LinearChainCRFGradOp : public framework::OperatorWithKernel { protected: // Explicitly set that the data type of output of the linear_chain_crf_grad // operator is determined by its input: gradients of LogLikelihood. - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType( - ctx.Input(framework::GradVarName("LogLikelihood"))->type()); + return framework::OpKernelType( + framework::ToDataType( + ctx.Input(framework::GradVarName("LogLikelihood")) + ->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/linear_chain_crf_op.h b/paddle/operators/linear_chain_crf_op.h index 56fb0c9102bee6e2fefd1180ef20237891573f70..ddf73981751798c72cef08f2dd5c87580b45aec3 100644 --- a/paddle/operators/linear_chain_crf_op.h +++ b/paddle/operators/linear_chain_crf_op.h @@ -134,7 +134,7 @@ class LinearChainCRFOpKernel : public framework::OpKernel { Tensor emission_row_max; emission_row_max.mutable_data( - framework::make_ddim({static_cast(batch_size), 1}), + framework::make_ddim({static_cast(batch_size), 1}), platform::CPUPlace()); auto place = ctx.GetEigenDevice(); @@ -273,7 +273,7 @@ class LinearChainCRFOpKernel : public framework::OpKernel { const int* lbl = label.data(); PADDLE_ENFORCE_LT( - *std::max_element(lbl, lbl + seq_length), tag_num, + static_cast(*std::max_element(lbl, lbl + seq_length)), tag_num, "An invalid tag label that execesses the largest tag number."); // Calculate the nominator part, which depends on the label sequence. diff --git a/paddle/operators/load_op.cc b/paddle/operators/load_op.cc index 2d4eff0c35af520dd27b9eb197937026a8fbdff9..b71a33a6b1ce80b545e6d7a4020dafc941dc55d2 100644 --- a/paddle/operators/load_op.cc +++ b/paddle/operators/load_op.cc @@ -115,14 +115,18 @@ class LoadOpProtoMaker : public framework::OpProtoAndCheckerMaker { LoadOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "The tensor need to be loaded"); - AddComment(R"DOC(Load Operator -Load operator will load a tensor variable from disk file. -)DOC"); + AddOutput("Out", "(Tensor) The tensor need to be loaded"); AddAttr("file_path", + "(string) " "Variable will be loaded from \"file_path\".") .AddCustomChecker( [](const std::string &path) { return !path.empty(); }); + AddComment(R"DOC( +Load Operator. + +Load operator will load a tensor variable from disk file. + +)DOC"); } }; } // namespace operators diff --git a/paddle/operators/lod_array_length_op.cc b/paddle/operators/lod_array_length_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..80445eb575703be3354595672a4c064b30e0f18c --- /dev/null +++ b/paddle/operators/lod_array_length_op.cc @@ -0,0 +1,71 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class LoDArrayLengthOp : public framework::OperatorBase { + public: + LoDArrayLengthOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + out.Resize({1}); + auto cpu = platform::CPUPlace(); + *out.mutable_data(cpu) = static_cast(x.size()); + } +}; + +class LoDArrayLengthProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDArrayLengthProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensorArray) The input tensor array."); + AddOutput("Out", "(Tensor) 1x1 CPU Tensor of length, int64_t"); + AddComment(R"DOC(Get the length of lod tensor array + +Out = len(X) + +NOTE: The output is a CPU Tensor since the control variable should be only in +CPU and the length of LoDTensorArray should be used as control variables. +)DOC"); + } +}; + +class LoDArrayLengthInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasOutput("Out")); + context->SetOutputDim("Out", {1}); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(lod_array_length, ops::LoDArrayLengthOp, + ops::LoDArrayLengthInferShape, ops::LoDArrayLengthProtoMaker, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/lod_rank_table_op.cc b/paddle/operators/lod_rank_table_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..f7d4db1947b83fecf57575e17fafe26795c92bdd --- /dev/null +++ b/paddle/operators/lod_rank_table_op.cc @@ -0,0 +1,82 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/op_registry.h" +namespace paddle { +namespace operators { + +class LoDRankTableOp : public framework::OperatorBase { + public: + LoDRankTableOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto x = scope.FindVar(Input("X"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + VLOG(10) << "Level = " << static_cast(Attr("level")); + out->Reset(x.lod(), static_cast(Attr("level"))); + } +}; + +class LoDRankTableOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDRankTableOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(LoDTensor) input lod tensor, must contain lod information."); + AddOutput("Out", "(LoDRankTable) The rank table of specific level."); + AddAttr("level", "(int) the specific lod level to rank.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment(R"DOC(Create LoDRanTable by LoDTensor + +LoD Rank Table stores the `level` of `lod` which is ordered by sequence +length in descending order. It is useful when implement dynamic RNN and is +shared by dynamic RNN memory, dynamic RNN slice input and dynamic RNN slice +output operators. +)DOC"); + } +}; + +class LoDRankTableInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), "LoDRankTable must has input X"); + } +}; + +class LoDRankTableInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind &op_desc, + framework::BlockDescBind *block) const override { + for (auto &o : op_desc.Output("Out")) { + block->FindRecursiveOrCreateVar(o)->SetType( + framework::VarDesc::LOD_RANK_TABLE); + } + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(lod_rank_table, paddle::operators::LoDRankTableOp, + paddle::operators::LoDRankTableOpProtoMaker, + paddle::operators::LoDRankTableInferShape, + paddle::operators::LoDRankTableInferVarType, + paddle::framework::EmptyGradOpMaker); diff --git a/paddle/operators/lod_reset_op.cc b/paddle/operators/lod_reset_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..32831cb1e2cf188a507773ef1e00b22de98d82ab --- /dev/null +++ b/paddle/operators/lod_reset_op.cc @@ -0,0 +1,120 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/lod_reset_op.h" + +namespace paddle { +namespace operators { + +class LoDResetOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + // input check + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of LoDResetOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of LoDResetOp should not be null."); + // If target LoD is not set form Input(), then it must be set from Attr(). + if (!ctx->HasInput("TargetLoD")) { + auto level0 = ctx->Attrs().Get>("target_lod"); + PADDLE_ENFORCE(level0.size() > 1, + "Target LoD is not found, should be set to be a valid one " + "through Input() or Attr()."); + } + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } +}; + +class LoDResetOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDResetOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensor) The input tensor of lod_reset operator."); + AddInput("TargetLoD", + "(Tensor, optional) The target level 0 LoD from Input().") + .AsDispensable(); + AddOutput("Out", "(LoDTensor) The output tensor of lod_reset operator."); + AddAttr>("target_lod", + "The target level 0 LoD from Attr().") + .SetDefault(std::vector{}); + AddComment(R"DOC(LoDReset operator + +Reset LoD of Input(X) into a new one specified by Input(TargetLoD) or +Attr(target_lod), or set LoD for Input(X) if it doesn't have one. +Currently the lod_reset operator only supports the reset of level 0 LoD. +At least one of Input(TargetLoD) and Attr(target_lod) must be set, +and if both of them are set, Input(TargetLoD) will be chosen as the +target LoD. + +An example: +Given a float LoDTensor X with shape (6, 1), its transpose form represents + + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + +with LoD = [[0, 2, 5, 6]] and the three (transposed) sequences look like + + [1.0, 2.0], [3.0, 4.0, 5.0], [6.0]. + +If target LoD = [0, 4, 6], the lod_reset operator will reset the LoD and +the sequences that the LoDTensor Output(Out) contains becomes: + + [1.0, 2.0, 3.0, 4.0], [5.0, 6.0]. + +)DOC"); + } +}; + +class LoDResetGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(lod_reset, ops::LoDResetOp, ops::LoDResetOpMaker, lod_reset_grad, + ops::LoDResetGradOp); +REGISTER_OP_CPU_KERNEL(lod_reset, + ops::LoDResetKernel, + ops::LoDResetKernel); +REGISTER_OP_CPU_KERNEL( + lod_reset_grad, ops::LoDResetGradKernel, + ops::LoDResetGradKernel); diff --git a/paddle/operators/lod_reset_op.cu b/paddle/operators/lod_reset_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..5244a17c3aad01909e3b8cf5f4d5abf8a44edc7f --- /dev/null +++ b/paddle/operators/lod_reset_op.cu @@ -0,0 +1,24 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/lod_reset_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(lod_reset, + ops::LoDResetKernel, + ops::LoDResetKernel); +REGISTER_OP_GPU_KERNEL( + lod_reset_grad, ops::LoDResetGradKernel, + ops::LoDResetGradKernel); diff --git a/paddle/operators/lod_reset_op.h b/paddle/operators/lod_reset_op.h new file mode 100644 index 0000000000000000000000000000000000000000..2bb916ccee80c83a02ea429fe95f5fafc86ccfa6 --- /dev/null +++ b/paddle/operators/lod_reset_op.h @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class LoDResetKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* out = ctx.Output("Out"); + auto* in = ctx.Input("X"); + auto* lod_t = ctx.Input("TargetLoD"); + + std::vector level0; + if (lod_t) { + auto* lod = lod_t->data(); + if (platform::is_gpu_place(ctx.GetPlace())) { + framework::Tensor lod_cpu; + lod_cpu.CopyFrom(*lod_t, platform::CPUPlace(), ctx.device_context()); + lod = lod_cpu.data(); + } + level0 = std::vector(lod, lod + lod_t->numel()); + } else { + level0 = ctx.Attr>("target_lod"); + } + + PADDLE_ENFORCE(level0.size() > 1UL, + "The size of target LoD should be greater than 1."); + PADDLE_ENFORCE(level0[0] == 0, + "Target LoD should be a vector starting from 0."); + PADDLE_ENFORCE(level0.back() == in->dims()[0], + "Target LoD should be a vector end with the " + "first dimension of Input(X)."); + for (size_t i = 0; i < level0.size() - 1; ++i) { + PADDLE_ENFORCE(level0[i + 1] > level0[i], + "Target LoD should be an ascending vector."); + } + + out->ShareDataWith(*in); + // cast level0 to size_t + std::vector ulevel0(level0.size(), 0); + std::transform(level0.begin(), level0.end(), ulevel0.begin(), + [](int a) { return static_cast(a); }); + framework::LoD target_lod; + target_lod.push_back(ulevel0); + out->set_lod(target_lod); + } +}; + +template +class LoDResetGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* d_out = ctx.Input(framework::GradVarName("Out")); + auto* d_x = ctx.Output(framework::GradVarName("X")); + + d_x->ShareDataWith(*d_out); + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/lod_tensor_to_array_op.cc b/paddle/operators/lod_tensor_to_array_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..58af35564d83b9699af4f7783fb6367ff9590682 --- /dev/null +++ b/paddle/operators/lod_tensor_to_array_op.cc @@ -0,0 +1,160 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/framework/lod_tensor_array.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +struct CopyRange { + size_t begin; + size_t end; +}; + +class LoDTensorToArrayOp : public framework::OperatorBase { + public: + LoDTensorToArrayOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &rank_table = + scope.FindVar(Input("RankTable"))->Get(); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + + auto &items = rank_table.items(); + auto max_seq_len = items[0].length; + auto rank_level = rank_table.level(); + out.resize(max_seq_len); + std::vector> copy_ranges(max_seq_len); + + // set out[i] lod + for (size_t t = 0; t < max_seq_len; t++) { + auto &lod = *out[t].mutable_lod(); + lod.clear(); + for (auto &item : items) { + if (t >= item.length) { + break; + } + size_t start_idx = x.lod()[rank_level][item.index] + t; + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + x.lod(), start_idx, start_idx + 1, rank_level + 1); + + auto &lod_length = lod_and_offset.first; + framework::AppendLoD(&lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + copy_ranges[t].emplace_back(CopyRange{start_offset, end_offset}); + } + } + + for (size_t i = 0; i < max_seq_len; ++i) { + auto &ranges = copy_ranges[i]; + size_t height = std::accumulate( + ranges.begin(), ranges.end(), 0UL, + [](size_t a, const CopyRange &b) { return a + b.end - b.begin; }); + auto x_dim = x.dims(); + x_dim[0] = static_cast(height); + out[i].Resize(x_dim); + out[i].mutable_data(x.place(), x.type()); + size_t offset = 0; + for (auto &each_range : ranges) { + size_t len = each_range.end - each_range.begin; + if (len == 0) { + continue; + } + // out[i][offset: offset+len] = x[each_range.begin: each_range.end] + out[i] + .Slice(static_cast(offset), static_cast(offset + len)) + .CopyFrom(x.Slice(static_cast(each_range.begin), + static_cast(each_range.end)), + x.place(), dev_ctx); + offset += len; + } + } + } +}; + +class LoDTensorToArrayOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + LoDTensorToArrayOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddInput("RankTable", ""); + AddOutput("Out", ""); + AddComment(""); + } +}; + +class LoDTensorToArrayInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "Input(X) of LoDTensorToArrayOp should not be null."); + PADDLE_ENFORCE( + context->HasInput("RankTable"), + "Input(RankTable) of LoDTensorToArrayOp should not be null."); + + PADDLE_ENFORCE(context->HasOutput("Out"), + "Output(Out) of LoDTensorToArrayOp should not be null."); + + auto x_dim = context->GetInputDim("X"); + // The first dim of each LoDTensor in Output can only be set at run-time.; + // We still have to Resize each LoDTensor in Output. + context->SetOutputDim("Out", x_dim); + } +}; + +class LoDTensorToArrayInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind &op_desc, + framework::BlockDescBind *block) const override { + for (auto &out_var : op_desc.Output("Out")) { + block->Var(out_var)->SetType(framework::VarDesc::LOD_TENSOR_ARRAY); + } + } +}; + +class LoDTensorToArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("array_to_lod_tensor"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("RankTable", Input("RankTable")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(lod_tensor_to_array, ops::LoDTensorToArrayOp, + ops::LoDTensorToArrayOpProtoMaker, + ops::LoDTensorToArrayInferShape, + ops::LoDTensorToArrayInferVarType, + ops::LoDTensorToArrayGradMaker); diff --git a/paddle/operators/lookup_table_op.cc b/paddle/operators/lookup_table_op.cc index 0b361e20f2037b9b75bc8670488dff1c50fb689c..93e812ac5be5aea6bf3ab353d31480322c51ccbc 100644 --- a/paddle/operators/lookup_table_op.cc +++ b/paddle/operators/lookup_table_op.cc @@ -41,9 +41,11 @@ class LookupTableOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("W")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("W")->type()), + ctx.device_context()); } }; @@ -53,21 +55,27 @@ class LookupTableOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("W", - "An input represents embedding tensors," - " which is a learnable parameter."); + "An input represents embedding tensors, " + "which is a learnable parameter."); AddInput("Ids", - "An input with type int32 or int64" - "contains the ids to be looked up in W." - "Ids must be a column vector with rank = 2." - "The 2nd dimension size must be 1"); - AddOutput("Out", "The lookup results, which have the same type with W."); - AddAttr("is_sparse", "Sparse update").SetDefault(false); + "An input with type int32 or int64 " + "contains the ids to be looked up in W. " + "Ids must be a column vector with rank = 2. " + "The 2nd dimension size must be 1."); + AddOutput("Out", "The lookup results, which have the same type as W."); + AddAttr("is_sparse", + "(boolean, default false) " + "Sparse update") + .SetDefault(false); AddComment(R"DOC( +Lookup Table Operator. + This operator is used to perform lookups on the parameter W, then concatenated into a dense tensor. -The input `Ids` can carry the LoD (Level of Details) information, -or not. And the output only shares the LoD with input `Ids`. +The input Ids can carry the LoD (Level of Details) information, +or not. And the output only shares the LoD information with input Ids. + )DOC"); } }; @@ -91,9 +99,11 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("W")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("W")->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/lookup_table_op.cu b/paddle/operators/lookup_table_op.cu index c7ba1720662fe80c945f2b4aa19745e408d40948..84b044184a36a0d3a72a4105d6baf401b4774cf7 100644 --- a/paddle/operators/lookup_table_op.cu +++ b/paddle/operators/lookup_table_op.cu @@ -74,10 +74,10 @@ class LookupTableCUDAKernel : public framework::OpKernel { dim3 threads(128, 8); dim3 grids(8, 1); - LookupTable<<< - grids, threads, 0, reinterpret_cast( - context.device_context()) - .stream()>>>(output, table, ids, N, K, D); + LookupTable< + T, 128, 8, + 8><<>>( + output, table, ids, N, K, D); } }; @@ -95,9 +95,7 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { auto* ids_data = ids->data(); auto ids_dim = ids->dims(); - auto stream = reinterpret_cast( - context.device_context()) - .stream(); + auto stream = context.cuda_device_context().stream(); // copy GPU memory to CPU pinned memory framework::Vector new_rows; new_rows.resize(ids_dim[0]); @@ -136,11 +134,10 @@ class LookupTableGradCUDAKernel : public framework::OpKernel { dim3 threads(128, 8); dim3 grids(8, 1); - LookupTableGrad<<( - context.device_context()) - .stream()>>>(d_table, d_output, ids, N, K, D); + LookupTableGrad< + T, 128, 8, + 8><<>>( + d_table, d_output, ids, N, K, D); } } }; diff --git a/paddle/operators/lookup_table_op.h b/paddle/operators/lookup_table_op.h index ea3289d2731a4b2098c3a199464559b0a0ce7202..99b912163b71594340d8917645dff107fd208aea 100644 --- a/paddle/operators/lookup_table_op.h +++ b/paddle/operators/lookup_table_op.h @@ -90,11 +90,13 @@ class LookupTableGradKernel : public framework::OpKernel { auto* d_output_data = d_output->data(); auto* d_table_data = d_table->mutable_data(context.GetPlace()); + memset(d_table_data, 0, d_table->numel() * sizeof(T)); + for (int64_t i = 0; i < ids->numel(); ++i) { PADDLE_ENFORCE_LT(ids_data[i], N); PADDLE_ENFORCE_GE(ids_data[i], 0); for (int j = 0; j < D; ++j) { - d_table_data[ids_data[i] * D + j] = d_output_data[i * D + j]; + d_table_data[ids_data[i] * D + j] += d_output_data[i * D + j]; } } } diff --git a/paddle/operators/lrn_op.cc b/paddle/operators/lrn_op.cc index 89ea6bfdbd9b78dd0a81fd5ba465d09549162eb5..00392b7967d020a7951a16a7850a2f08735baeb8 100644 --- a/paddle/operators/lrn_op.cc +++ b/paddle/operators/lrn_op.cc @@ -45,72 +45,70 @@ class LRNOpMaker : public framework::OpProtoAndCheckerMaker { public: LRNOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", R"DOC( - (Tensor) The input of LRN operator. It must be a 4D tenor with NCHW format. - )DOC"); - + AddInput("X", + "(Tensor) The input of LRN operator. " + "It must be a 4D tenor with NCHW format."); AddOutput("Out", "(Tensor) The output of LRN operator, which is also the 4D " "tensor with NCHW format."); - AddOutput("MidOut", R"Doc( -(Tensor)Middle result of lrn op.It's computed in forward process -and also used in backward process. - )Doc"); - - AddAttr("n", R"DOC( -(int, default 5)n is “adjacent” kernel maps at the same spatial position. - )DOC") + AddOutput("MidOut", + "(Tensor) Middle result of LRN operator. It's computed in " + "forward process and also used in backward process."); + + AddAttr("n", + "(int default 5) " + "n is the \"adjacent\" kernel that maps " + "at the same spatial position.") .SetDefault(5) .GreaterThan(0); - AddAttr("k", R"DOC( -(float, default 2.0)k is the bias. - )DOC") + AddAttr("k", + "(float, default 2.0) " + "k is the bias.") .SetDefault(2.0) .GreaterThan(0.0); - AddAttr("alpha", R"DOC( -(float, default 0.0001)alpha is the scale number. - )DOC") + AddAttr("alpha", + "(float, default 0.0001) " + "alpha is the scale number.") .SetDefault(0.0001) .GreaterThan(0.0); - AddAttr("beta", R"DOC( -(float, default 0.75)beta is the power number. - )DOC") + AddAttr("beta", + "(float, default 0.75) " + "beta is the power number.") .SetDefault(0.75) .GreaterThan(0.0); AddComment(R"DOC( - Local Response Normalization. - - This Function comes from the paper - "ImageNet Classification with Deep Convolutional Neural Networks". +Local Response Normalization Operator. - The original formula is: +This operator comes from the paper +"ImageNet Classification with Deep Convolutional Neural Networks". - Input(i, x, y) - Output(i, x, y) = ---------------------------------------------- - -- upper - (k + alpha * > (Input(j, x, y))^2) ^ (beta) - -- j = lower +The original formula is: - upper is `min(C, c + n/2)` - lower if `max(0, c - n/2)` +$$ +Output(i, x, y) = Input(i, x, y) / \left( +k + \alpha \sum\limits^{\min(C, c + n/2)}_{j = \max(0, c - n/2)} +(Input(j, x, y))^2 +\right)^{\beta} +$$ - Function implementation: +Function implementation: - inputs and outpus is NCHW format, while input.shape.ndims() is equal 4. - And the meaning of each dimension(0-3) is respectively batch size, - feature maps, rows and columns. +Inputs and outpus are in NCHW format, while input.shape.ndims() equals 4. +And dimensions 0 ~ 3 represent batch size, feature maps, rows, +and columns, respectively. - Input and Output in the above formula is for each map(i) of one image, and - Input(i, x, y), Output(i, x, y) represents an element in an image. +Input and Output in the formula above is for each map(i) of one image, and +Input(i, x, y), Output(i, x, y) represents an element in an image. - C is the number of feature maps of one image, and n is a hyper-parameters - is configured when Function is initialized. The sum in the denominator - is the sum of the same position in the neighboring maps. - )DOC"); +C is the number of feature maps of one image. n is a hyper-parameter +configured when operator is initialized. The sum in the denominator +is the sum of the same positions in the neighboring maps. + +)DOC"); } }; diff --git a/paddle/operators/lstm_op.cc b/paddle/operators/lstm_op.cc index 94342d940704d850a2a45c281a3d88de5a132753..4cbb60f3fdab968e8c36d4fbad55fd3efc7b1d0d 100644 --- a/paddle/operators/lstm_op.cc +++ b/paddle/operators/lstm_op.cc @@ -24,6 +24,11 @@ class LSTMOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Input"), "Input(Input) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Bias"), + "Input(Bias) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Hidden"), "Output(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Cell"), @@ -59,11 +64,13 @@ class LSTMOp : public framework::OperatorWithKernel { "The second dimension of Input(Weight) " "should be 4 * %d.", frame_size); + auto b_dims = ctx->GetInputDim("Bias"); PADDLE_ENFORCE_EQ(b_dims.size(), 2, "The rank of Input(Bias) should be 2."); PADDLE_ENFORCE_EQ(b_dims[0], 1, "The first dimension of Input(Bias) should be 1."); - if (ctx->Attrs().Get("usePeepholes")) { + + if (ctx->Attrs().Get("use_peepholes")) { PADDLE_ENFORCE_EQ(b_dims[1], 7 * frame_size, "The second dimension of Input(Bias) should be " "7 * %d if enable peepholes connection", @@ -74,6 +81,7 @@ class LSTMOp : public framework::OperatorWithKernel { "4 * %d if disable peepholes connection", frame_size); } + framework::DDim out_dims({in_dims[0], frame_size}); ctx->SetOutputDim("Hidden", out_dims); ctx->SetOutputDim("Cell", out_dims); @@ -84,10 +92,11 @@ class LSTMOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType( - ctx.Input("Input")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Input")->type()), + ctx.device_context()); } }; @@ -103,7 +112,7 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("H0", "(Tensor, optional) the initial hidden state is an optional " "input. This is a tensor with shape (N x D), where N is the " - "batch size, D is the hidden size.") + "batch size and D is the hidden size.") .AsDispensable(); AddInput("C0", "(Tensor, optional) the initial cell state is an optional " @@ -117,14 +126,13 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Bias", "(Tensor) the learnable weights, which contains two parts: " "input-hidden bias weight and peephole connections weight if " - "setting `usePeepholes` True. " - "1. `usePeepholes = False` " + "setting `use_peepholes` True. " + "1. `use_peepholes = False` " " - The shape is (1 x 4D). " " - Bias = {b_c, b_i, b_f, b_o}." - "2. `usePeepholes = True` " + "2. `use_peepholes = True` " " - The shape is (1 x 7D). " - " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}.") - .AsDispensable(); + " - Bias = {b_c, b_i, b_f, b_o, W_ic, W_fc, W_oc}."); AddOutput("Hidden", "(LoDTensor) the hidden state of LSTM operator. " "The shape is (T x D), and lod is the same with the `Input`."); @@ -134,85 +142,85 @@ class LSTMOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("BatchGate", "(LoDTensor) This LoDTensor contains input gate, forget gate " "and output gate after the nonlinear computation. This " - "LoDTensor has the same shape with the reorganized input, which " + "LoDTensor has the same shape as the reorganized input, which " "is also be called batch input. The LoD size is 2. The first " "LoD is the batch offsets and the second LoD contains the " "indexes, which denote the position of reorganized sequence " "in the raw input.") .AsIntermediate(); AddOutput("BatchCellPreAct", - "(LoDTensor) This LoDTensor is got in the forward and used " + "(LoDTensor) This LoDTensor is obtained in the forward and used " "in the backward.") .AsIntermediate(); - AddAttr("usePeepholes", + AddAttr("use_peepholes", "(bool, defalut: True) " "whether to enable diagonal/peephole connections.") .SetDefault(true); - AddAttr("isReverse", + AddAttr("is_reverse", "(bool, defalut: False) " "whether to compute reversed LSTM.") .SetDefault(false); AddAttr( - "gateActivation", + "gate_activation", "(string, default: sigmoid)" "The activation for input gate, forget gate and output " "gate, `sigmoid` by default.") - .SetDefault("sigmoid"); - AddAttr("cellActivation", + .SetDefault("sigmoid") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); + AddAttr("cell_activation", "(string, default: tanh)" "The activation for cell output, `tanh` by defalut.") - .SetDefault("tanh"); - AddAttr("candidateActivation", + .SetDefault("tanh") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); + AddAttr("candidate_activation", "(string, default: tanh)" "The activation for candidate hidden state, " "`tanh` by default.") - .SetDefault("tanh"); - AddComment(R"DOC(Long-Short Term Memory (LSTM) Operator + .SetDefault("tanh") + .InEnum({"sigmoid", "tanh", "relu", "identity"}); + AddComment(R"DOC( +Long-Short Term Memory (LSTM) Operator. -The defalut implementation is diagonal/peephole connection [1], the formula is -as follows +The defalut implementation is diagonal/peephole connection +(https://arxiv.org/pdf/1402.1128.pdf), the formula is as follows: - i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) +$$ +i_t = \sigma(W_{ix}x_{t} + W_{ih}h_{t-1} + W_{ic}c_{t-1} + b_i) \\ - f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) +f_t = \sigma(W_{fx}x_{t} + W_{fh}h_{t-1} + W_{fc}c_{t-1} + b_f) \\ - \tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) +\tilde{c_t} = act_g(W_{cx}x_t + W_{ch}h_{t-1} + b_c) \\ - o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) +o_t = \sigma(W_{ox}x_{t} + W_{oh}h_{t-1} + W_{oc}c_t + b_o) \\ - c_t = f_t ⊙ c_{t-1} + i_t ⊙ \tilde{c_t} +c_t = f_t \odot c_{t-1} + i_t \odot \tilde{c_t} \\ - h_t = o_t ⊙ act_h(c_t) +h_t = o_t \odot act_h(c_t) +$$ where the W terms denote weight matrices (e.g. \f$W_{xi}\f$ is the matrix of weights from the input gate to the input), \f$W_{ic}, W_{fc}, W_{oc}\f$ -are diagonal weight matrices for peephole connections. In our implenmention, -We use vectors to reprenset these diagonal weight matrices. The b terms +are diagonal weight matrices for peephole connections. In our implementation, +we use vectors to reprenset these diagonal weight matrices. The b terms denote bias vectors (\f$b_i\f$ is the input gate bias vector), \f$\sigma\f$ -is the non-line actications, such as logistic sigmoid function, and -\f$i, f, o\f$ and \f$c\f$ are respectively the input gate, forget gate, -output gate and cell activation vectors, all of which are the same size as +is the non-line activations, such as logistic sigmoid function, and +\f$i, f, o\f$ and \f$c\f$ are the input gate, forget gate, output gate, +and cell activation vectors, respectively, all of which have the same size as the cell output activation vector \f$h\f$. -The ⊙ is the element-wise product of the vectors, \f$act_g\f$ and \f$act_h\f$ -are the cell input and cell output activation functions, `tanh` is usually +The \f$\odot\f$ is the element-wise product of the vectors. \f$act_g\f$ and \f$act_h\f$ +are the cell input and cell output activation functions and `tanh` is usually used for them. \f$\tilde{c_t}\f$ is also called candidate hidden state, which is computed based on the current input and the previous hidden state. -Set `usePeepholes` False to disable peephole connection [2]. The formula +Set `use_peepholes` False to disable peephole connection +(http://www.bioinf.jku.at/publications/older/2604.pdf). The formula is omitted here. -@note These \f$W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}\f$ -operations on the input x_{t} were NOT included in this operator. +Note that these \f$W_{xi}x_{t}, W_{xf}x_{t}, W_{xc}x_{t}, W_{xo}x_{t}\f$ +operations on the input \f$x_{t}\f$ are NOT included in this operator. Users can choose to use fully-connect operator before LSTM operator. -[1] Hasim Sak, Andrew Senior, and Francoise Beaufays. Long short-term memory -recurrent neural network architectures for large scale acoustic modeling. -INTERSPEECH, 2014. - -[2] S. Hochreiter and J. Schmidhuber. Long Short-Term Memory. -Neural Computation, 9(8):1735-1780, 1997. - )DOC"); } }; @@ -228,30 +236,35 @@ class LSTMGradOp : public framework::OperatorWithKernel { "Input(Hidden) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("Cell"), "Input(Cell) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Weight"), + "Input(Weight) of LSTM should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Bias"), + "Input(Bias) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("BatchGate"), "Input(BatchGate) of LSTM should not be null."); PADDLE_ENFORCE(ctx->HasInput("BatchCellPreAct"), "Input(BatchGate) of LSTM should not be null."); - auto in_g_name = framework::GradVarName("Input"); - if (ctx->HasOutput(in_g_name)) - ctx->SetOutputDim(in_g_name, ctx->GetInputDim("Input")); - - auto w_g_name = framework::GradVarName("Weight"); - if (ctx->HasOutput(w_g_name)) - ctx->SetOutputDim(w_g_name, ctx->GetInputDim("Weight")); - - auto b_g_name = framework::GradVarName("Bias"); - if (ctx->HasOutput(b_g_name)) - ctx->SetOutputDim(b_g_name, ctx->GetInputDim("Bias")); + auto SetOutGradDim = [&ctx](const std::string& name) { + auto g_name = framework::GradVarName(name); + if (ctx->HasOutput(g_name)) + ctx->SetOutputDim(g_name, ctx->GetInputDim(name)); + }; + + SetOutGradDim("Input"); + SetOutGradDim("Weight"); + SetOutGradDim("Bias"); + SetOutGradDim("H0"); + SetOutGradDim("C0"); } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType( - ctx.Input("Input")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Input")->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/lstm_op.h b/paddle/operators/lstm_op.h index af088b80b4283cf221a1dff74546d73d977fada3..fca84e2d8fa832a3780eab7e0fa2facceb4d613b 100644 --- a/paddle/operators/lstm_op.h +++ b/paddle/operators/lstm_op.h @@ -28,6 +28,15 @@ template using EigenMatrix = framework::EigenMatrix; +template +inline void ReorderInitState(const platform::DeviceContext& ctx, + const framework::Tensor& src, const size_t* index, + framework::Tensor* dst, bool indexed_src) { + math::CopyMatrixRowsFunctor row_shuffle; + dst->mutable_data(src.dims(), ctx.GetPlace()); + row_shuffle(ctx, src, index, *dst, indexed_src); +} + template class LSTMKernel : public framework::OpKernel { public: @@ -36,6 +45,9 @@ class LSTMKernel : public framework::OpKernel { auto* weight = ctx.Input("Weight"); auto* bias = ctx.Input("Bias"); + auto* hidden_t0 = ctx.Input("H0"); + auto* cell_t0 = ctx.Input("C0"); + auto* batch_gate = ctx.Output("BatchGate"); batch_gate->mutable_data(ctx.GetPlace()); auto* hidden_out = ctx.Output("Hidden"); @@ -43,12 +55,7 @@ class LSTMKernel : public framework::OpKernel { auto* cell_out = ctx.Output("Cell"); cell_out->mutable_data(ctx.GetPlace()); - // Now the function ShareLoD in InferShape is not implemented. - // So copy LoD here. - ctx.ShareLoD("Input", "Hidden"); - ctx.ShareLoD("Input", "Cell"); - - bool is_reverse = ctx.Attr("isReverse"); + bool is_reverse = ctx.Attr("is_reverse"); math::LoDTensor2BatchFunctor to_batch; auto& device_ctx = ctx.device_context(); to_batch(device_ctx, *input, *batch_gate, true, is_reverse); @@ -71,7 +78,7 @@ class LSTMKernel : public framework::OpKernel { } math::LstmMetaValue lstm_value; - if (bias) { + if (bias && ctx.Attr("use_peepholes")) { T* bias_data = const_cast(bias->data()); // the code style in LstmMetaValue will be updated later. @@ -84,6 +91,16 @@ class LSTMKernel : public framework::OpKernel { lstm_value.checkOg = nullptr; } lstm_value.prevStateValue = nullptr; + Tensor ordered_c0; + const size_t* order = batch_gate->lod()[2].data(); + if (cell_t0) { + // Since the batch computing for LSTM reorders the input sequence + // according to their length. The initialized cell state also needs + // to reorder. + ReorderInitState(device_ctx, *cell_t0, order, &ordered_c0, + true); + lstm_value.prevStateValue = ordered_c0.data(); + } // Use the local variable as here. LoDTensor batch_hidden, batch_cell; @@ -94,9 +111,9 @@ class LSTMKernel : public framework::OpKernel { auto batch_starts = batch_gate->lod()[0]; size_t num_batch = batch_starts.size() - 1; - auto gate_act = ctx.Attr("gateActivation"); - auto cell_act = ctx.Attr("cellActivation"); - auto cand_act = ctx.Attr("candidateActivation"); + auto gate_act = ctx.Attr("gate_activation"); + auto cell_act = ctx.Attr("cell_activation"); + auto cand_act = ctx.Attr("candidate_activation"); for (size_t n = 0; n < num_batch; n++) { int bstart = static_cast(batch_starts[n]); @@ -109,15 +126,28 @@ class LSTMKernel : public framework::OpKernel { int cur_batch_size = bend - bstart; - if (n != 0) { + if (n > 0) { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_t = batch_hidden.Slice(pre_h_start, pre_h_end); math::matmul(device_ctx, pre_hidden_t, false, *weight, false, static_cast(1.0), &gate_t, static_cast(1.0)); + } else if (hidden_t0) { + // If n == 0 and there is no initialized hidden state, that is to say + // the H0 is zeros, the calculation W_h * H0 will be skiped. + // If n == 0 and there is initialized hidden state, calculate W_h * H0. + + // Since the batch computing for LSTM reorders the input sequence + // according to their length. The initialized hidden state also needs + // to reorder. + Tensor ordered_h0; + ReorderInitState(device_ctx, *hidden_t0, order, &ordered_h0, + true); + math::matmul(device_ctx, ordered_h0, false, *weight, false, + static_cast(1.0), &gate_t, + static_cast(1.0)); } - // else if : FIXME support the initial hidden and cell lstm_value.gateValue = gate_t.data(); lstm_value.outputValue = out_t.data(); @@ -160,6 +190,12 @@ class LSTMGradKernel : public framework::OpKernel { auto* weight_g = ctx.Output(framework::GradVarName("Weight")); auto* bias_g = ctx.Output(framework::GradVarName("Bias")); + auto* h0 = ctx.Input("H0"); + auto* c0 = ctx.Input("C0"); + + auto* h0_g = ctx.Output(framework::GradVarName("H0")); + auto* c0_g = ctx.Output(framework::GradVarName("C0")); + auto& device_ctx = ctx.device_context(); math::SetConstant zero; if (weight_g) { @@ -167,13 +203,25 @@ class LSTMGradKernel : public framework::OpKernel { zero(device_ctx, weight_g, static_cast(0.0)); } + // ordered_h0/c0 is the reordered hidden/cell initialization. + // ordered_h0_g/c0_g is the reordered gradient of hidden/cell + // initialization. + Tensor ordered_h0, ordered_c0, ordered_h0_g, ordered_c0_g; + const size_t* order = batch_gate->lod()[2].data(); + if (c0) { + ReorderInitState(device_ctx, *c0, order, &ordered_c0, true); + } + if (c0 && c0_g) { + ordered_c0_g.mutable_data(c0_g->dims(), ctx.GetPlace()); + } + auto in_dims = input->dims(); auto out_dims = hidden_g->dims(); int frame_size = static_cast(in_dims[1] / 4); PADDLE_ENFORCE_EQ(frame_size, out_dims[1]); math::LstmMetaValue lstm_value; - if (bias) { + if (bias && ctx.Attr("use_peepholes")) { T* bias_data = const_cast(bias->data()); lstm_value.checkIg = bias_data + 4 * frame_size; lstm_value.checkFg = lstm_value.checkIg + frame_size; @@ -185,9 +233,13 @@ class LSTMGradKernel : public framework::OpKernel { } math::LstmMetaGrad lstm_grad; + if (bias && bias_g) { - T* bias_g_data = const_cast(bias_g->mutable_data(ctx.GetPlace())); + bias_g->mutable_data(ctx.GetPlace()); zero(device_ctx, bias_g, static_cast(0.0)); + } + if (bias && bias_g && ctx.Attr("use_peepholes")) { + T* bias_g_data = bias_g->data(); lstm_grad.checkIgGrad = bias_g_data + 4 * frame_size; lstm_grad.checkFgGrad = lstm_grad.checkIgGrad + frame_size; lstm_grad.checkOgGrad = lstm_grad.checkFgGrad + frame_size; @@ -199,36 +251,30 @@ class LSTMGradKernel : public framework::OpKernel { math::LoDTensor2BatchFunctor to_batch; - // use the local variable as here. - LoDTensor batch_hidden; - batch_hidden.mutable_data(out_dims, ctx.GetPlace()); - batch_hidden.set_lod(batch_gate->lod()); - to_batch(device_ctx, *hidden_out, batch_hidden, false); + auto ToBatch = [&batch_gate, &to_batch]( + const platform::DeviceContext& ctx, const framework::LoDTensor& src, + const framework::DDim& dims, framework::LoDTensor& dst) { + dst.mutable_data(dims, ctx.GetPlace()); + dst.set_lod(batch_gate->lod()); + to_batch(ctx, src, dst, false); + }; - LoDTensor batch_hidden_g; - batch_hidden_g.mutable_data(out_dims, ctx.GetPlace()); - batch_hidden_g.set_lod(batch_gate->lod()); - to_batch(device_ctx, *hidden_g, batch_hidden_g, false); + LoDTensor batch_hidden, batch_hidden_g, batch_cell; + ToBatch(device_ctx, *hidden_out, out_dims, batch_hidden); + ToBatch(device_ctx, *hidden_g, out_dims, batch_hidden_g); + ToBatch(device_ctx, *cell_out, out_dims, batch_cell); - LoDTensor batch_cell; - batch_cell.mutable_data(out_dims, ctx.GetPlace()); - batch_cell.set_lod(batch_gate->lod()); - to_batch(device_ctx, *cell_out, batch_cell, false); - - LoDTensor batch_cell_g; + LoDTensor batch_cell_g, batch_gate_g; batch_cell_g.mutable_data(out_dims, ctx.GetPlace()); - batch_cell_g.set_lod(batch_gate->lod()); // TODO(qingqing) support the case output cell has gradient. // to_batch(device_ctx, *cell_g, batch_cell_g, false); zero(device_ctx, &batch_cell_g, static_cast(0.0)); - - LoDTensor batch_gate_g; batch_gate_g.mutable_data(batch_gate->dims(), ctx.GetPlace()); batch_gate_g.set_lod(batch_gate->lod()); - auto gate_act = ctx.Attr("gateActivation"); - auto cell_act = ctx.Attr("cellActivation"); - auto cand_act = ctx.Attr("candidateActivation"); + auto gate_act = ctx.Attr("gate_activation"); + auto cell_act = ctx.Attr("cell_activation"); + auto cand_act = ctx.Attr("candidate_activation"); auto batch_starts = batch_gate->lod()[0]; size_t num_batch = batch_starts.size() - 1; @@ -250,15 +296,15 @@ class LSTMGradKernel : public framework::OpKernel { lstm_grad.gateGrad = gate_g.data(); lstm_grad.outputGrad = out_g.data(); - if (n) { + if (n > 0) { int bstart_pre = static_cast(batch_starts[n - 1]); Tensor cell_pre = batch_cell.Slice(bstart_pre, bstart); Tensor cell_pre_g = batch_cell_g.Slice(bstart_pre, bstart); lstm_value.prevStateValue = cell_pre.data(); lstm_grad.prevStateGrad = cell_pre_g.data(); } else { - lstm_value.prevStateValue = nullptr; - lstm_grad.prevStateGrad = nullptr; + lstm_value.prevStateValue = c0 ? ordered_c0.data() : nullptr; + lstm_grad.prevStateGrad = c0_g ? ordered_c0_g.data() : nullptr; } int cur_batch_size = bend - bstart; @@ -266,7 +312,7 @@ class LSTMGradKernel : public framework::OpKernel { device_ctx, lstm_value, lstm_grad, frame_size, cur_batch_size, gate_act, cell_act, cand_act); - if (n != 0) { + if (n > 0) { int pre_h_start = static_cast(batch_starts[n - 1]); int pre_h_end = pre_h_start + cur_batch_size; auto pre_hidden_g = batch_hidden_g.Slice(pre_h_start, pre_h_end); @@ -280,6 +326,19 @@ class LSTMGradKernel : public framework::OpKernel { static_cast(1.0), weight_g, static_cast(1.0)); } + } else { + if (h0 && weight_g) { + ReorderInitState(device_ctx, *h0, order, &ordered_h0, true); + math::matmul(device_ctx, ordered_h0, true, gate_g, false, + static_cast(1.0), weight_g, + static_cast(1.0)); + } + if (h0 && h0_g) { + ordered_h0_g.mutable_data(h0_g->dims(), ctx.GetPlace()); + math::matmul(device_ctx, gate_g, false, *weight, true, + static_cast(1.0), &ordered_h0_g, + static_cast(0.0)); + } } } @@ -302,6 +361,13 @@ class LSTMGradKernel : public framework::OpKernel { math::gemv(device_ctx, true, m, n, 1., batch_gate_g.data(), ones.data(), 0., bias_g->data()); } + + if (h0 && h0_g) { + ReorderInitState(device_ctx, ordered_h0_g, order, h0_g, false); + } + if (c0 && c0_g) { + ReorderInitState(device_ctx, ordered_c0_g, order, c0_g, false); + } } }; diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc index 5d63017208a55ec4bcc2e8d66f1ca2e1b84d4593..18b9cdf2a39e8226c634194ff2cc56d169979774 100644 --- a/paddle/operators/lstm_unit_op.cc +++ b/paddle/operators/lstm_unit_op.cc @@ -34,10 +34,10 @@ class LstmUnitOp : public framework::OperatorWithKernel { auto c_prev_dims = ctx->GetInputDim("C_prev"); PADDLE_ENFORCE_EQ(x_dims.size(), 2, "Input(X)'s rank must be 2."); - PADDLE_ENFORCE(x_dims[0] == c_prev_dims[0], - "Batch size of inputs and states must be equal"); - PADDLE_ENFORCE(x_dims[1] == c_prev_dims[1] * 4, - "Dimension of FC should equal to prev state * 4"); + PADDLE_ENFORCE_EQ(x_dims[0], c_prev_dims[0], + "Batch size of inputs and states must be equal"); + PADDLE_ENFORCE_EQ(x_dims[1], c_prev_dims[1] * 4, + "Dimension of FC should equal to prev state * 4"); int b_size = c_prev_dims[0]; // batch size int s_dim = c_prev_dims[1]; // state dim @@ -57,17 +57,22 @@ class LstmUnitOpMaker : public framework::OpProtoAndCheckerMaker { "The cell state tensor of last time-step in the Lstm Unit operator."); AddOutput("C", "The cell tensor of Lstm Unit operator."); AddOutput("H", "The hidden state tensor of Lstm Unit operator."); - - AddComment(R"DOC(Lstm-Unit Operator + AddAttr("forget_bias", + "(float, default 0.0) " + "The forget bias of Lstm Unit.") + .SetDefault(0.0); + AddComment(R"DOC( +Lstm Unit Operator Equation: - i, f, o, j = split(X) - C = C_prev * sigm(f + forget_bias) + sigm(i) * tanh(j) - H = C * sigm(o) + +$$ +i, f, o, j = split(X) \\ +C = C_{prev} * sigm(f + forget\_bias) + sigm(i) * tanh(j) \\ +H = C * sigm(o) +$$ )DOC"); - AddAttr("forget_bias", "The forget bias of Lstm Unit.") - .SetDefault(0.0); } }; diff --git a/paddle/operators/lstm_unit_op.cu b/paddle/operators/lstm_unit_op.cu index 49ea550b6f49a13bf31d14321d7a9eb13a834d4b..e192283aa0afac49e8e467506f3703d1ce60d2a6 100644 --- a/paddle/operators/lstm_unit_op.cu +++ b/paddle/operators/lstm_unit_op.cu @@ -12,6 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* Acknowledgement: the following code is strongly inspired by +https://github.com/caffe2/caffe2/blob/master/caffe2/operators/lstm_unit_op_gpu.cu +*/ + #include "paddle/framework/op_registry.h" #include "paddle/operators/cross_entropy_op.h" #include "paddle/platform/assert.h" diff --git a/paddle/operators/lstm_unit_op.h b/paddle/operators/lstm_unit_op.h index 625b1852c2f0eb2ed435f73fea251c40c614a7dd..38cb298f92a21bb5c7508761fec701d28279a85f 100644 --- a/paddle/operators/lstm_unit_op.h +++ b/paddle/operators/lstm_unit_op.h @@ -12,6 +12,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* Acknowledgement: the following code is strongly inspired by +https://github.com/caffe2/caffe2/blob/master/caffe2/operators/lstm_unit_op.h +*/ + #pragma once #include "glog/logging.h" #include "paddle/framework/op_registry.h" diff --git a/paddle/operators/margin_rank_loss_op.cc b/paddle/operators/margin_rank_loss_op.cc index 638a99addc2119e8f44648cc54b97bd8a892d2bc..d7e8a0ea7632650203106b01531d724cf0b8e085 100644 --- a/paddle/operators/margin_rank_loss_op.cc +++ b/paddle/operators/margin_rank_loss_op.cc @@ -55,8 +55,6 @@ class MarginRankLossOpMaker : public framework::OpProtoAndCheckerMaker { "(2-D tensor with shape [batch_size x 1]) " "The label indicating X1 ranked higher than X2 or not, " "can only be +1 or -1."); - AddAttr("margin", "(scalar, default 0) Margin for MarginRankLossOp.") - .SetDefault(static_cast(0)); AddOutput("Activated", "(2-D tensor with shape [batch_size x 1]) Intermediate tensor " "to indicate whether each element of Output(Out) is activated.") @@ -64,23 +62,26 @@ class MarginRankLossOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(2-D tensor with shape [batch_size x 1]) " "The output loss of MarginRankLoss operator."); + AddAttr("margin", "(scalar, default 0) Margin for MarginRankLossOp.") + .SetDefault(static_cast(0)); AddComment(R"DOC( +MarginRankLoss Operator. -MarginRankLoss operator measures the loss given a pair of training sample +This operator measures the loss given a pair of training sample {`X1`, `X2`} and the `Label` with attribute `margin`, where `Label = +1` -indicating X1 is ranked higher than `X2`, otherwise `Label = -1`. The loss -turns out +indicating X1 is ranked higher than `X2` and `Label = -1` otherwise. The loss +is calculated as: -loss(X1, X2, Label) = max(0, -Label * (X1 - X2) + margin). +$loss(X1, X2, Label) = \max(0, -Label * (X1 - X2) + margin)$ -The attribute `margin` involved here helps make the predictions more robust. +The attribute `margin` here helps make the predictions more robust. Denote the item ranked higher as the positive sample, otherwise the negative sample. If the score of the two samples satisfies -positive sample - negative sample < margin, +$positive sample - negative sample < margin$ -the pair of samples will contribute to the final loss, which will backpropogate -and train the ranking model to enlarge the difference of the two score. +the pair of samples will contribute to the final loss, which will backpropagate +and train the ranking model to enlarge the difference between the two scores. For batch input with size `batch_size`, `X1`, `X2` and `Label` all have the same shape [batch_size x 1]. diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index 40cc177d0f19c2359626ef972e787a0b1c5580f8..ab7f23f57043844d45c36acc475422613164bee1 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -8,20 +8,24 @@ if(WITH_GPU) nv_library(softmax SRCS softmax.cc softmax.cu DEPS operator) nv_library(cross_entropy SRCS cross_entropy.cc cross_entropy.cu DEPS operator) nv_library(pooling SRCS pooling.cc pooling.cu DEPS device_context) + nv_library(sequence_pooling SRCS sequence_pooling.cc sequence_pooling.cu DEPS device_context math_function) nv_library(vol2col SRCS vol2col.cc vol2col.cu DEPS device_context) nv_library(context_project SRCS context_project.cc context_project.cu DEPS device_context) nv_library(sequence2batch SRCS sequence2batch.cc sequence2batch.cu DEPS device_context) nv_library(lstm_compute SRCS lstm_compute.cc lstm_compute.cu DEPS device_context activation_functions) + nv_library(gru_compute SRCS gru_compute.cc gru_compute.cu DEPS device_context activation_functions math_function) else() cc_library(math_function SRCS math_function.cc im2col.cc DEPS cblas device_context operator) cc_library(selected_rows_functor SRCS selected_rows_functor.cc DEPS selected_rows math_function) cc_library(softmax SRCS softmax.cc DEPS operator) cc_library(cross_entropy SRCS cross_entropy.cc DEPS operator) cc_library(pooling SRCS pooling.cc DEPS device_context) + cc_library(sequence_pooling SRCS sequence_pooling.cc DEPS device_context math_function) cc_library(vol2col SRCS vol2col.cc DEPS device_context) cc_library(context_project SRCS context_project.cc DEPS device_context) cc_library(sequence2batch SRCS sequence2batch.cc DEPS device_context) cc_library(lstm_compute SRCS lstm_compute.cc DEPS device_context activation_functions) + cc_library(gru_compute SRCS gru_compute.cc DEPS device_context activation_functions math_function) endif() cc_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) diff --git a/paddle/operators/math/detail/CMakeLists.txt b/paddle/operators/math/detail/CMakeLists.txt index 49cf228de2204cb4888cf645a0cb68ed04cc3371..0df1c060f9042067b655d987560a278f9fc46a5b 100644 --- a/paddle/operators/math/detail/CMakeLists.txt +++ b/paddle/operators/math/detail/CMakeLists.txt @@ -1,5 +1 @@ -if(WITH_AVX) - cc_library(activation_functions SRCS hl_cpu_functions.cc hl_avx_functions.cc) -else() - cc_library(activation_functions SRCS hl_cpu_functions.cc) -endif() +cc_library(activation_functions SRCS avx_functions.cc) diff --git a/paddle/operators/math/detail/activation_functions.h b/paddle/operators/math/detail/activation_functions.h new file mode 100644 index 0000000000000000000000000000000000000000..a20c35d1d9dc4a3a6fae92023fd1aae787a716ec --- /dev/null +++ b/paddle/operators/math/detail/activation_functions.h @@ -0,0 +1,170 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/platform/hostdevice.h" + +#ifdef __AVX__ +#include +#endif + +namespace paddle { +namespace operators { +namespace math { +namespace detail { + +#define SIGMOID_THRESHOLD_MIN -40.0 +#define SIGMOID_THRESHOLD_MAX 13.0 +#define EXP_MAX_INPUT 40.0 + +namespace forward { + +template +DEVICE T Identity(const T a) { + return a; +} + +template +DEVICE T Relu(const T a) { + return a > static_cast(0.0) ? a : static_cast(0.0); +} + +template +DEVICE T Sigmoid(const T a) { + const T min = SIGMOID_THRESHOLD_MIN; + const T max = SIGMOID_THRESHOLD_MAX; + T tmp = (a < min) ? min : ((a > max) ? max : a); + return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); +} + +template +DEVICE T Tanh(const T a) { + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + return (2.0 / (1.0 + exp(tmp))) - 1.0; +} + +} // namespace forward + +namespace backward { + +template +DEVICE T Identity(const T a, const T b) { + return a; +} + +template +DEVICE T Relu(const T a, const T b) { + return a * (b > 0.0 ? 1.0 : 0.0); +} + +template +DEVICE T Sigmoid(const T a, const T b) { + return a * b * (1.0 - b); +} + +template +DEVICE T Tanh(const T a, const T b) { + return a * (1.0 - b * b); +} + +} // namespace backward + +template +struct Active { + typedef T (*Act)(T); + typedef T (*ActGrad)(T, T); +}; + +static DEVICE Active::Act kActFloat[] = { + &forward::Sigmoid, &forward::Relu, &forward::Tanh, + &forward::Identity}; + +static DEVICE Active::ActGrad kActGradFloat[] = { + &backward::Sigmoid, &backward::Relu, &backward::Tanh, + &backward::Identity}; + +static DEVICE Active::Act kActDouble[] = { + &forward::Sigmoid, &forward::Relu, &forward::Tanh, + &forward::Identity}; + +static DEVICE Active::ActGrad kActGradDouble[] = { + &backward::Sigmoid, &backward::Relu, + &backward::Tanh, &backward::Identity}; + +namespace forward { +inline DEVICE float activation(float a, int index) { + return kActFloat[index](a); +} + +inline DEVICE double activation(double a, int index) { + return kActDouble[index](a); +} + +} // namespace forward + +namespace backward { +inline DEVICE float activation(float a, float b, int index) { + return kActGradFloat[index](a, b); +} + +inline DEVICE double activation(double a, double b, int index) { + return kActGradDouble[index](a, b); +} +} // namespace backward + +#ifdef __AVX__ +namespace forward { +namespace avx { +__m256 Relu(const __m256 a); +__m256 Sigmoid(const __m256 a); +__m256 Tanh(const __m256 a); +__m256 Identity(const __m256 a); +} // namespace avx +} // namespace forward + +namespace backward { +namespace avx { +__m256 Relu(const __m256 a, const __m256 b); +__m256 Sigmoid(const __m256 a, const __m256 b); +__m256 Tanh(const __m256 a, const __m256 b); +__m256 Identity(const __m256 a, const __m256 b); +} // namespace avx +} // namespace backward + +static Active<__m256>::Act kActAvx[] = { + &forward::avx::Sigmoid, &forward::avx::Relu, &forward::avx::Tanh, + &forward::avx::Identity}; + +static Active<__m256>::ActGrad kActGradAvx[] = { + &backward::avx::Sigmoid, &backward::avx::Relu, &backward::avx::Tanh, + &backward::avx::Identity}; + +namespace forward { +inline __m256 activation(__m256 a, int index) { return kActAvx[index](a); } +} // namespace forward + +namespace backward { +inline __m256 activation(__m256 a, __m256 b, int index) { + return kActGradAvx[index](a, b); +} +} // namespace backward + +#endif + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/hl_avx_functions.cc b/paddle/operators/math/detail/avx_functions.cc similarity index 68% rename from paddle/operators/math/detail/hl_avx_functions.cc rename to paddle/operators/math/detail/avx_functions.cc index 415bac5d93ee00244d072b0998c6941b14d4f8d8..921364788cd23e265fa0ca027bf1af3f81604489 100644 --- a/paddle/operators/math/detail/hl_avx_functions.cc +++ b/paddle/operators/math/detail/avx_functions.cc @@ -12,59 +12,79 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#ifdef __AVX__ + #include -#include "hl_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" // TODO(qingqing) refine this dependence #include "paddle/cuda/src/avx_mathfun.h" -namespace hppl { +namespace paddle { +namespace operators { +namespace math { +namespace detail { -__m256 exp(__m256 a) { return exp256_ps(a); } +__m256 Exp(__m256 a) { return exp256_ps(a); } -__m256 relu(const __m256 a) { +namespace forward { +namespace avx { +__m256 Relu(const __m256 a) { __m256 tmp = _mm256_set1_ps(0.0f); return _mm256_max_ps(a, tmp); } -__m256 sigmoid(const __m256 a) { +__m256 Sigmoid(const __m256 a) { __m256 max = _mm256_set1_ps(SIGMOID_THRESHOLD_MAX); __m256 min = _mm256_set1_ps(SIGMOID_THRESHOLD_MIN); __m256 tmp = _mm256_max_ps(a, min); tmp = _mm256_min_ps(tmp, max); tmp = _mm256_sub_ps(_mm256_set1_ps(0.0f), tmp); - tmp = exp(tmp); + tmp = Exp(tmp); tmp = _mm256_add_ps(_mm256_set1_ps(1.0f), tmp); tmp = _mm256_div_ps(_mm256_set1_ps(1.0f), tmp); return tmp; } -__m256 tanh(const __m256 a) { +__m256 Tanh(const __m256 a) { __m256 max = _mm256_set1_ps(EXP_MAX_INPUT); __m256 tmp = _mm256_mul_ps(_mm256_set1_ps(-2.0f), a); tmp = _mm256_min_ps(tmp, max); - tmp = exp(tmp); + tmp = Exp(tmp); return _mm256_sub_ps(_mm256_div_ps(_mm256_set1_ps(2.0f), _mm256_add_ps(_mm256_set1_ps(1.0f), tmp)), _mm256_set1_ps(1.0f)); } -__m256 linear(const __m256 a) { return a; } +__m256 Identity(const __m256 a) { return a; } + +} // namespace avx +} // namespace forward -__m256 relu(const __m256 a, const __m256 b) { +namespace backward { +namespace avx { +__m256 Relu(const __m256 a, const __m256 b) { return _mm256_mul_ps( a, _mm256_and_ps(_mm256_cmp_ps(b, _mm256_set1_ps(0.0f), _CMP_GT_OS), _mm256_set1_ps(1.0f))); } -__m256 sigmoid(const __m256 a, const __m256 b) { +__m256 Sigmoid(const __m256 a, const __m256 b) { return _mm256_mul_ps(_mm256_mul_ps(a, b), _mm256_sub_ps(_mm256_set1_ps(1.0f), b)); } -__m256 tanh(const __m256 a, const __m256 b) { +__m256 Tanh(const __m256 a, const __m256 b) { return _mm256_mul_ps( a, _mm256_sub_ps(_mm256_set1_ps(1.0f), _mm256_mul_ps(b, b))); } -__m256 linear(const __m256 a, const __m256 b) { return a; } -} // namespace hppl +__m256 Identity(const __m256 a, const __m256 b) { return a; } +} // namespace avx +} // namespace backward + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle + +#endif diff --git a/paddle/operators/math/detail/gru_cpu_kernel.h b/paddle/operators/math/detail/gru_cpu_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..51af140cf4d5e6581765bea00033fa53d383230d --- /dev/null +++ b/paddle/operators/math/detail/gru_cpu_kernel.h @@ -0,0 +1,424 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/operators/math/detail/activation_functions.h" +#include "paddle/operators/math/gru_compute.h" + +namespace paddle { +namespace operators { +namespace math { +namespace detail { + +#ifndef __NVCC__ + +template +void hl_naive_gru_forward_reset_output(OpResetOutput opResetOutput, + T *gateValue, T *resetOutputValue, + T *prevOutputValue, int frameSize, + activation_mode_t active_gate) { + T rValueUpdateGate; + T rValueResetGate; + T rValueResetOutput; + T rPrevOut = 0; + T *updateGate = gateValue; + T *resetGate = gateValue + frameSize; + + for (int i = 0; i < frameSize; i++) { + rValueUpdateGate = updateGate[i]; + rValueResetGate = resetGate[i]; + if (prevOutputValue) { + rPrevOut = prevOutputValue[i]; + } + + opResetOutput(rValueUpdateGate, rValueResetGate, rPrevOut, + rValueResetOutput, active_gate); + + updateGate[i] = rValueUpdateGate; + resetGate[i] = rValueResetGate; + resetOutputValue[i] = rValueResetOutput; + } +} + +template +void hl_naive_gru_forward_final_output(OpFinalOutput opFinalOutput, + T *gateValue, T *prevOutputValue, + T *outputValue, int frameSize, + activation_mode_t active_node) { + T rValueUpdateGate; + T rValueFrameState; + T rPrevOut = 0; + T rOutput; + T *updateGate = gateValue; + T *frameState = gateValue + frameSize * 2; + + for (int i = 0; i < frameSize; i++) { + rValueUpdateGate = updateGate[i]; + rValueFrameState = frameState[i]; + if (prevOutputValue) { + rPrevOut = prevOutputValue[i]; + } + + opFinalOutput(rValueUpdateGate, rValueFrameState, rPrevOut, rOutput, + active_node); + + frameState[i] = rValueFrameState; + outputValue[i] = rOutput; + } +} + +template +void hl_avx_gru_forward_reset_output(OpResetOutput opResetOutput, T *gateValue, + T *resetOutputValue, T *prevOutputValue, + int frameSize, + activation_mode_t active_gate) { +#ifdef __AVX__ + __m256 rValueUpdateGate; + __m256 rValueResetGate; + __m256 rValueResetOutput; + __m256 rPrevOut = _mm256_set1_ps(0.0f); + __m256 *updateGate = (__m256 *)gateValue; + __m256 *resetGate = (__m256 *)(gateValue + frameSize); + + for (int i = 0; i < frameSize / 8; i++) { + rValueUpdateGate = updateGate[i]; + rValueResetGate = resetGate[i]; + if (prevOutputValue) { + rPrevOut = ((__m256 *)prevOutputValue)[i]; + } + + opResetOutput(rValueUpdateGate, rValueResetGate, rPrevOut, + rValueResetOutput, active_gate); + + updateGate[i] = rValueUpdateGate; + resetGate[i] = rValueResetGate; + ((__m256 *)resetOutputValue)[i] = rValueResetOutput; + } +#endif +} + +template +void hl_avx_gru_forward_final_output(OpFinalOutput opFinalOutput, T *gateValue, + T *prevOutputValue, T *outputValue, + int frameSize, + activation_mode_t active_node) { +#ifdef __AVX__ + __m256 rValueUpdateGate; + __m256 rValueFrameState; + __m256 rPrevOut = _mm256_set1_ps(0.0f); + __m256 rOutput; + __m256 *updateGate = (__m256 *)gateValue; + __m256 *frameState = (__m256 *)(gateValue + frameSize * 2); + + for (int i = 0; i < frameSize / 8; i++) { + rValueUpdateGate = updateGate[i]; + rValueFrameState = frameState[i]; + if (prevOutputValue) { + rPrevOut = ((__m256 *)prevOutputValue)[i]; + } + + opFinalOutput(rValueUpdateGate, rValueFrameState, rPrevOut, rOutput, + active_node); + + frameState[i] = rValueFrameState; + ((__m256 *)outputValue)[i] = rOutput; + } +#endif +} + +template +inline void forward_reset_output(OpResetOutput opResetOutput, + hl_gru_value value, int frameSize, + int batchSize, activation_mode_t active_gate) { + for (int b = 0; b < batchSize; b++) { + if (OpResetOutput::avx && !(frameSize & (8 - 1)) && (sizeof(T) == 4)) { + hl_avx_gru_forward_reset_output( + opResetOutput, value.gateValue, value.resetOutputValue, + value.prevOutValue, frameSize, active_gate); + } else { + hl_naive_gru_forward_reset_output( + opResetOutput, value.gateValue, value.resetOutputValue, + value.prevOutValue, frameSize, active_gate); + } + + value.gateValue += frameSize * 3; + value.resetOutputValue += frameSize; + if (value.prevOutValue) { + value.prevOutValue += frameSize; + } + } +} + +template +inline void forward_final_output(OpFinalOutput opFinalOutput, + hl_gru_value value, int frameSize, + int batchSize, activation_mode_t active_node) { + for (int b = 0; b < batchSize; b++) { + if (OpFinalOutput::avx && !(frameSize & (8 - 1)) && (sizeof(T) == 4)) { + hl_avx_gru_forward_final_output(opFinalOutput, value.gateValue, + value.prevOutValue, value.outputValue, + frameSize, active_node); + } else { + hl_naive_gru_forward_final_output(opFinalOutput, value.gateValue, + value.prevOutValue, value.outputValue, + frameSize, active_node); + } + + value.gateValue += frameSize * 3; + value.outputValue += frameSize; + if (value.prevOutValue) { + value.prevOutValue += frameSize; + } + } +} + +template +void hl_naive_gru_backward_state_grad(OpStateGrad opStateGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *outputGrad, + int frameSize, + activation_mode_t active_node) { + T rUpdateGateValue; + T rUpdateGateGrad; + T rFrameStateValue; + T rFrameStateGrad; + T rOutGrad; + T rPrevOutValue = 0; + T rPrevOutGrad = 0; + T *updateGateValue = gateValue; + T *updateGateGrad = gateGrad; + T *frameStateValue = gateValue + frameSize * 2; + T *frameStateGrad = gateGrad + frameSize * 2; + + for (int i = 0; i < frameSize; i++) { + rUpdateGateValue = updateGateValue[i]; + rFrameStateValue = frameStateValue[i]; + rOutGrad = outputGrad[i]; + if (prevOutValue) { + rPrevOutValue = prevOutValue[i]; + } + if (prevOutGrad) { + rPrevOutGrad = prevOutGrad[i]; + } + + opStateGrad(rUpdateGateValue, rUpdateGateGrad, rFrameStateValue, + rFrameStateGrad, rPrevOutValue, rPrevOutGrad, rOutGrad, + active_node); + + updateGateGrad[i] = rUpdateGateGrad; + frameStateGrad[i] = rFrameStateGrad; + if (prevOutGrad) { + prevOutGrad[i] = rPrevOutGrad; + } + } +} + +template +void hl_naive_gru_backward_reset_grad(OpResetGrad opResetGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *resetOutputGrad, + int frameSize, + activation_mode_t active_gate) { + T rUpdateGateValue; + T rUpdateGateGrad; + T rResetGateValue; + T rResetGateGrad; + T rResetOutputGrad = 0; + T rPrevOutValue = 0; + T rPrevOutGrad = 0; + T *updateGateValue = gateValue; + T *updateGateGrad = gateGrad; + T *resetGateValue = gateValue + frameSize; + T *resetGateGrad = gateGrad + frameSize; + + for (int i = 0; i < frameSize; i++) { + rUpdateGateValue = updateGateValue[i]; + rUpdateGateGrad = updateGateGrad[i]; + rResetGateValue = resetGateValue[i]; + + if (prevOutValue && prevOutGrad) { + rResetOutputGrad = resetOutputGrad[i]; + } + if (prevOutValue) { + rPrevOutValue = prevOutValue[i]; + } + if (prevOutGrad) { + rPrevOutGrad = prevOutGrad[i]; + } + + opResetGrad(rUpdateGateValue, rUpdateGateGrad, rResetGateValue, + rResetGateGrad, rPrevOutValue, rPrevOutGrad, rResetOutputGrad, + active_gate); + + updateGateGrad[i] = rUpdateGateGrad; + resetGateGrad[i] = rResetGateGrad; + if (prevOutGrad) { + prevOutGrad[i] = rPrevOutGrad; + } + } +} + +template +void hl_avx_gru_backward_state_grad(OpStateGrad opStateGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *outputGrad, + int frameSize, + activation_mode_t active_node) { +#ifdef __AVX__ + __m256 rUpdateGateValue; + __m256 rUpdateGateGrad; + __m256 rFrameStateValue; + __m256 rFrameStateGrad; + __m256 rOutGrad; + __m256 rPrevOutValue = _mm256_set1_ps(0.0f); + __m256 rPrevOutGrad = _mm256_set1_ps(0.0f); + __m256 *updateGateValue = (__m256 *)gateValue; + __m256 *updateGateGrad = (__m256 *)gateGrad; + __m256 *frameStateValue = (__m256 *)(gateValue + frameSize * 2); + __m256 *frameStateGrad = (__m256 *)(gateGrad + frameSize * 2); + + for (int i = 0; i < frameSize / 8; i++) { + rUpdateGateValue = updateGateValue[i]; + rFrameStateValue = frameStateValue[i]; + rOutGrad = ((__m256 *)outputGrad)[i]; + if (prevOutValue) { + rPrevOutValue = ((__m256 *)prevOutValue)[i]; + } + if (prevOutGrad) { + rPrevOutGrad = ((__m256 *)prevOutGrad)[i]; + } + + opStateGrad(rUpdateGateValue, rUpdateGateGrad, rFrameStateValue, + rFrameStateGrad, rPrevOutValue, rPrevOutGrad, rOutGrad, + active_node); + + updateGateGrad[i] = rUpdateGateGrad; + frameStateGrad[i] = rFrameStateGrad; + if (prevOutGrad) { + ((__m256 *)prevOutGrad)[i] = rPrevOutGrad; + } + } +#endif +} + +template +void hl_avx_gru_backward_reset_grad(OpResetGrad opResetGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *resetOutputGrad, + int frameSize, + activation_mode_t active_gate) { +#ifdef __AVX__ + __m256 rUpdateGateValue; + __m256 rUpdateGateGrad; + __m256 rResetGateValue; + __m256 rResetGateGrad; + __m256 rResetOutputGrad = _mm256_set1_ps(0.0f); + __m256 rPrevOutValue = _mm256_set1_ps(0.0f); + __m256 rPrevOutGrad = _mm256_set1_ps(0.0f); + __m256 *updateGateValue = (__m256 *)gateValue; + __m256 *updateGateGrad = (__m256 *)gateGrad; + __m256 *resetGateValue = (__m256 *)(gateValue + frameSize); + __m256 *resetGateGrad = (__m256 *)(gateGrad + frameSize); + + for (int i = 0; i < frameSize / 8; i++) { + rUpdateGateValue = updateGateValue[i]; + rUpdateGateGrad = updateGateGrad[i]; + rResetGateValue = resetGateValue[i]; + + if (prevOutValue && prevOutGrad) { + rResetOutputGrad = ((__m256 *)resetOutputGrad)[i]; + } + if (prevOutValue) { + rPrevOutValue = ((__m256 *)prevOutValue)[i]; + } + if (prevOutGrad) { + rPrevOutGrad = ((__m256 *)prevOutGrad)[i]; + } + + opResetGrad(rUpdateGateValue, rUpdateGateGrad, rResetGateValue, + rResetGateGrad, rPrevOutValue, rPrevOutGrad, rResetOutputGrad, + active_gate); + + updateGateGrad[i] = rUpdateGateGrad; + resetGateGrad[i] = rResetGateGrad; + if (prevOutGrad) { + ((__m256 *)prevOutGrad)[i] = rPrevOutGrad; + } + } +#endif +} + +template +inline void backward_state_grad(OpStateGrad opStateGrad, hl_gru_value value, + hl_gru_grad grad, int frameSize, + int batchSize, activation_mode_t active_node) { + for (int b = 0; b < batchSize; b++) { + if (OpStateGrad::avx && !(frameSize & (8 - 1)) && (sizeof(T) == 4)) { + hl_avx_gru_backward_state_grad( + opStateGrad, value.gateValue, grad.gateGrad, value.prevOutValue, + grad.prevOutGrad, grad.outputGrad, frameSize, active_node); + } else { + hl_naive_gru_backward_state_grad( + opStateGrad, value.gateValue, grad.gateGrad, value.prevOutValue, + grad.prevOutGrad, grad.outputGrad, frameSize, active_node); + } + + value.gateValue += frameSize * 3; + if (value.prevOutValue) { + value.prevOutValue += frameSize; + } + + grad.gateGrad += frameSize * 3; + grad.outputGrad += frameSize; + if (grad.prevOutGrad) { + grad.prevOutGrad += frameSize; + } + } +} + +template +inline void backward_reset_grad(OpResetGrad opResetGrad, hl_gru_value value, + hl_gru_grad grad, int frameSize, + int batchSize, activation_mode_t active_gate) { + for (int b = 0; b < batchSize; b++) { + if (OpResetGrad::avx && !(frameSize & (8 - 1)) && (sizeof(T) == 4)) { + hl_avx_gru_backward_reset_grad( + opResetGrad, value.gateValue, grad.gateGrad, value.prevOutValue, + grad.prevOutGrad, grad.resetOutputGrad, frameSize, active_gate); + } else { + hl_naive_gru_backward_reset_grad( + opResetGrad, value.gateValue, grad.gateGrad, value.prevOutValue, + grad.prevOutGrad, grad.resetOutputGrad, frameSize, active_gate); + } + + value.gateValue += frameSize * 3; + if (value.prevOutValue) { + value.prevOutValue += frameSize; + } + + grad.gateGrad += frameSize * 3; + grad.resetOutputGrad += frameSize; + if (grad.prevOutGrad) { + grad.prevOutGrad += frameSize; + } + } +} + +#endif + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/gru_gpu_kernel.h b/paddle/operators/math/detail/gru_gpu_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..6441c648b048422c110872a85aa8cb719f11a8d7 --- /dev/null +++ b/paddle/operators/math/detail/gru_gpu_kernel.h @@ -0,0 +1,203 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/operators/math/detail/activation_functions.h" +#include "paddle/operators/math/gru_compute.h" +#include "paddle/platform/cuda_helper.h" +#include "paddle/platform/device_context.h" + +#include + +namespace paddle { +namespace operators { +namespace math { +namespace detail { + +/* + * threads(framePerBlock, batchPerBlock) + * grid(frameBlocks, batchBlocks) + */ +template +__global__ void KeGruForwardResetOutput(OpResetOutput opResetOutput, + T *gateValue, T *resetOutputValue, + T *prevOutputValue, int frameSize, + int batchSize, + activation_mode_t active_gate) { + const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; + if (frameIdx >= frameSize) return; + + int batchIdx = 0; + if (isBatch) { + batchIdx = blockIdx.y * blockDim.y + threadIdx.y; + if (batchIdx >= batchSize) return; + gateValue += batchIdx * 3 * frameSize; + resetOutputValue += batchIdx * frameSize; + } + + T rPrevOut = 0; + T rValueResetOutput; + T rValueUpdateGate = gateValue[frameIdx + frameSize * 0]; + T rValueResetGate = gateValue[frameIdx + frameSize * 1]; + + if (prevOutputValue) { + if (isBatch) prevOutputValue += batchIdx * frameSize; + rPrevOut = prevOutputValue[frameIdx]; + } + + opResetOutput(rValueUpdateGate, rValueResetGate, rPrevOut, rValueResetOutput, + active_gate); + + gateValue[frameIdx + frameSize * 0] = rValueUpdateGate; + gateValue[frameIdx + frameSize * 1] = rValueResetGate; + resetOutputValue[frameIdx] = rValueResetOutput; +} + +/* + * threads(framePerBlock, batchPerBlock) + * grid(frameBlocks, batchBlocks) + */ +template +__global__ void KeGruForwardFinalOutput(OpFinalOutput opFinalOutput, + T *gateValue, T *prevOutputValue, + T *outputValue, int frameSize, + int batchSize, + activation_mode_t active_node) { + const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; + if (frameIdx >= frameSize) return; + int batchIdx = 0; + if (isBatch) { + batchIdx = blockIdx.y * blockDim.y + threadIdx.y; + if (batchIdx >= batchSize) return; + gateValue += batchIdx * 3 * frameSize; + outputValue += batchIdx * frameSize; + } + + T rOutput; + T rPrevOut = 0; + T rValueUpdateGate = gateValue[frameIdx + frameSize * 0]; + T rValueFrameState = gateValue[frameIdx + frameSize * 2]; + + if (prevOutputValue) { + if (isBatch) prevOutputValue += batchIdx * frameSize; + rPrevOut = prevOutputValue[frameIdx]; + } + + opFinalOutput(rValueUpdateGate, rValueFrameState, rPrevOut, rOutput, + active_node); + + gateValue[frameIdx + frameSize * 2] = rValueFrameState; + outputValue[frameIdx] = rOutput; +} + +/* + * threads(framePerBlock, batchPerBlock) + * grid(frameBlocks, batchBlocks) + */ +template +__global__ void KeGruBackwardStateGrad(OpStateGrad opStateGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *outputGrad, + int frameSize, int batchSize, + activation_mode_t active_node) { + const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; + if (frameIdx >= frameSize) return; + int batchIdx = 0; + if (isBatch) { + batchIdx = blockIdx.y * blockDim.y + threadIdx.y; + if (batchIdx >= batchSize) return; + gateValue += batchIdx * 3 * frameSize; + gateGrad += batchIdx * 3 * frameSize; + outputGrad += batchIdx * frameSize; + } + + T rUpdateGateGrad; + T rFrameStateGrad; + T rPrevOutValue = 0; + T rPrevOutGrad = 0; + T rUpdateGateValue = gateValue[frameIdx + frameSize * 0]; + T rFrameStateValue = gateValue[frameIdx + frameSize * 2]; + T rOutGrad = outputGrad[frameIdx]; + + if (prevOutValue && prevOutGrad) { + if (isBatch) prevOutValue += batchIdx * frameSize; + rPrevOutValue = prevOutValue[frameIdx]; + + if (isBatch) prevOutGrad += batchIdx * frameSize; + rPrevOutGrad = prevOutGrad[frameIdx]; + } + + opStateGrad(rUpdateGateValue, rUpdateGateGrad, rFrameStateValue, + rFrameStateGrad, rPrevOutValue, rPrevOutGrad, rOutGrad, + active_node); + + gateGrad[frameIdx + frameSize * 0] = rUpdateGateGrad; + gateGrad[frameIdx + frameSize * 2] = rFrameStateGrad; + if (prevOutGrad) { + prevOutGrad[frameIdx] = rPrevOutGrad; + } +} + +/* + * threads(framePerBlock, batchPerBlock) + * grid(frameBlocks, batchBlocks) + */ +template +__global__ void KeGruBackwardResetGrad(OpResetGrad opResetGrad, T *gateValue, + T *gateGrad, T *prevOutValue, + T *prevOutGrad, T *resetOutputGrad, + int frameSize, int batchSize, + activation_mode_t active_gate) { + const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; + if (frameIdx >= frameSize) return; + int batchIdx = 0; + if (isBatch) { + batchIdx = blockIdx.y * blockDim.y + threadIdx.y; + if (batchIdx >= batchSize) return; + gateValue += batchIdx * 3 * frameSize; + gateGrad += batchIdx * 3 * frameSize; + resetOutputGrad += batchIdx * frameSize; + } + + T rResetGateGrad; + T rPrevOutValue = 0; + T rPrevOutGrad = 0; + T rResetOutputGrad = 0; + T rUpdateGateValue = gateValue[frameIdx + frameSize * 0]; + T rUpdateGateGrad = gateGrad[frameIdx + frameSize * 0]; + T rResetGateValue = gateValue[frameIdx + frameSize * 1]; + + if (prevOutValue && prevOutGrad) { + if (isBatch) prevOutValue += batchIdx * frameSize; + if (isBatch) prevOutGrad += batchIdx * frameSize; + rPrevOutValue = prevOutValue[frameIdx]; + rPrevOutGrad = prevOutGrad[frameIdx]; + rResetOutputGrad = resetOutputGrad[frameIdx]; + } + + opResetGrad(rUpdateGateValue, rUpdateGateGrad, rResetGateValue, + rResetGateGrad, rPrevOutValue, rPrevOutGrad, rResetOutputGrad, + active_gate); + + gateGrad[frameIdx + frameSize * 0] = rUpdateGateGrad; + gateGrad[frameIdx + frameSize * 1] = rResetGateGrad; + if (prevOutGrad) { + prevOutGrad[frameIdx] = rPrevOutGrad; + } +} +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/gru_kernel.h b/paddle/operators/math/detail/gru_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..8a681d8d8bced72e1296f863489f6ccbc7913167 --- /dev/null +++ b/paddle/operators/math/detail/gru_kernel.h @@ -0,0 +1,155 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/detail/activation_functions.h" +#include "paddle/platform/hostdevice.h" + +#include + +// TODO(guosheng): refine code style in gru_kernel +namespace paddle { +namespace operators { +namespace math { +namespace detail { + +namespace forward { + +template +class gru_resetOutput { + public: + HOSTDEVICE void operator()(T &valueUpdateGate, T &valueResetGate, T &prevOut, + T &valueResetOutput, activation_mode_t actGate) { + valueUpdateGate = activation(valueUpdateGate, actGate); + valueResetGate = activation(valueResetGate, actGate); + valueResetOutput = prevOut * valueResetGate; + } +#ifndef __NVCC__ +#ifndef __AVX__ + static const bool avx = false; +#else + static const bool avx = true; + HOSTDEVICE void operator()(__m256 &valueUpdateGate, __m256 &valueResetGate, + __m256 &prevOut, __m256 &valueResetOutput, + activation_mode_t actGate) { + valueUpdateGate = activation(valueUpdateGate, actGate); + valueResetGate = activation(valueResetGate, actGate); + valueResetOutput = _mm256_mul_ps(prevOut, valueResetGate); + } +#endif +#endif +}; + +template +class gru_finalOutput { + public: + HOSTDEVICE void operator()(T &valueUpdateGate, T &valueFrameState, T &prevOut, + T &valueOutput, activation_mode_t actInput) { + valueFrameState = activation(valueFrameState, actInput); + valueOutput = prevOut - (valueUpdateGate * prevOut) + + (valueUpdateGate * valueFrameState); + } +#ifndef __NVCC__ +#ifndef __AVX__ + static const bool avx = false; +#else + static const bool avx = true; + HOSTDEVICE void operator()(__m256 &valueUpdateGate, __m256 &valueFrameState, + __m256 &prevOut, __m256 &valueOutput, + activation_mode_t actInput) { + valueFrameState = activation(valueFrameState, actInput); + valueOutput = _mm256_add_ps( + _mm256_sub_ps(prevOut, _mm256_mul_ps(valueUpdateGate, prevOut)), + _mm256_mul_ps(valueUpdateGate, valueFrameState)); + } +#endif +#endif +}; +} // namespace forward + +namespace backward { + +template +class gru_stateGrad { + public: + HOSTDEVICE void operator()(T &valueUpdateGate, T &gradUpdateGate, + T &valueFrameState, T &gradFrameState, + T &valuePrevOut, T &gradPrevOut, T &gradOutput, + activation_mode_t actInput) { + gradUpdateGate = (gradOutput * valueFrameState); + gradUpdateGate -= (gradOutput * valuePrevOut); + gradPrevOut -= (gradOutput * valueUpdateGate); + gradPrevOut += gradOutput; + gradFrameState = + activation(gradOutput * valueUpdateGate, valueFrameState, actInput); + } +#ifndef __NVCC__ +#ifndef __AVX__ + static const bool avx = false; +#else + static const bool avx = true; + HOSTDEVICE void operator()(__m256 &valueUpdateGate, __m256 &gradUpdateGate, + __m256 &valueFrameState, __m256 &gradFrameState, + __m256 &valuePrevOut, __m256 &gradPrevOut, + __m256 &gradOutput, activation_mode_t actInput) { + gradUpdateGate = _mm256_mul_ps(gradOutput, valueFrameState); + gradUpdateGate = + _mm256_sub_ps(gradUpdateGate, _mm256_mul_ps(gradOutput, valuePrevOut)); + gradPrevOut = _mm256_add_ps( + _mm256_sub_ps(gradPrevOut, _mm256_mul_ps(gradOutput, valueUpdateGate)), + gradOutput); + gradFrameState = activation(_mm256_mul_ps(gradOutput, valueUpdateGate), + valueFrameState, actInput); + } +#endif +#endif +}; + +template +class gru_resetGrad { + public: + HOSTDEVICE void operator()(T &valueUpdateGate, T &gradUpdateGate, + T &valueResetGate, T &gradResetGate, + T &valuePrevOut, T &gradPrevOut, + T &gradResetOutput, activation_mode_t actGate) { + gradResetGate = (gradResetOutput * valuePrevOut); + gradPrevOut += (gradResetOutput * valueResetGate); + gradUpdateGate = activation(gradUpdateGate, valueUpdateGate, actGate); + gradResetGate = activation(gradResetGate, valueResetGate, actGate); + } +#ifndef __NVCC__ +#ifndef __AVX__ + static const bool avx = false; +#else + static const bool avx = true; + HOSTDEVICE void operator()(__m256 &valueUpdateGate, __m256 &gradUpdateGate, + __m256 &valueResetGate, __m256 &gradResetGate, + __m256 &valuePrevOut, __m256 &gradPrevOut, + __m256 &gradResetOutput, + activation_mode_t actGate) { + gradResetGate = _mm256_mul_ps(gradResetOutput, valuePrevOut); + gradPrevOut = _mm256_add_ps(gradPrevOut, + _mm256_mul_ps(gradResetOutput, valueResetGate)); + gradUpdateGate = activation(gradUpdateGate, valueUpdateGate, actGate); + gradResetGate = activation(gradResetGate, valueResetGate, actGate); + } +#endif +#endif +}; + +} // namespace backward + +} // namespace detail +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/detail/hl_activation_functions.h b/paddle/operators/math/detail/hl_activation_functions.h deleted file mode 100644 index 9d7d9914f0090bff17049038dfa2288d84f3dbda..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_activation_functions.h +++ /dev/null @@ -1,188 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_ACTIVATION_FUNCTIONS_H_ -#define HL_ACTIVATION_FUNCTIONS_H_ - -#include "hl_functions.h" -#include "paddle/operators/math/lstm_compute.h" - -/** - * Active functions: sigmoid, relu, tanh and linear. - */ -#define FLOAT_ACTIVE_FUNCTION \ - { \ - hppl::typef::sigmoid, hppl::typef::relu, hppl::typef::tanh, \ - hppl::typef::linear \ - } - -#define DOUBLE_ACTIVE_FUNCTION \ - { \ - hppl::typed::sigmoid, hppl::typed::relu, hppl::typed::tanh, \ - hppl::typed::linear \ - } - -#define AVX_ACTIVE_FUNCTION \ - { hppl::sigmoid, hppl::relu, hppl::tanh, hppl::linear } - -namespace hppl { - -using activation_mode_t = paddle::operators::math::activation_mode_t; - -/** - * Hppl supports sigmoid, relu, tanh, linear active functions - * for neural networks' forward and backward activation. - */ -template -class Active { - public: - typedef T (*forward)(T); - typedef T (*backward)(T, T); -}; - -template -struct ForwardActType; - -template <> -struct ForwardActType { - using type = Active::forward; -}; - -template <> -struct ForwardActType { - using type = Active::forward; -}; - -template -struct BackwardActType; - -template <> -struct BackwardActType { - using type = Active::backward; -}; - -template <> -struct BackwardActType { - using type = Active::backward; -}; - -#ifdef __NVCC__ -namespace gpu { -static __device__ Active::forward forward[] = FLOAT_ACTIVE_FUNCTION; -static __device__ Active::backward backward[] = FLOAT_ACTIVE_FUNCTION; - -static __device__ Active::forward forward_d[] = DOUBLE_ACTIVE_FUNCTION; -static __device__ Active::backward backward_d[] = - DOUBLE_ACTIVE_FUNCTION; - -template -struct ForwardAct { - __device__ typename ForwardActType::type operator()( - activation_mode_t type); -}; - -template <> -struct ForwardAct { - __device__ ForwardActType::type operator()(activation_mode_t type) { - return forward[type]; - } -}; - -template <> -struct ForwardAct { - __device__ ForwardActType::type operator()(activation_mode_t type) { - return forward_d[type]; - } -}; - -template -struct BackwardAct { - __device__ typename BackwardActType::type operator()( - activation_mode_t type); -}; - -template <> -struct BackwardAct { - __device__ BackwardActType::type operator()(activation_mode_t type) { - return backward[type]; - } -}; - -template <> -struct BackwardAct { - __device__ BackwardActType::type operator()(activation_mode_t type) { - return backward_d[type]; - } -}; - -} // namespace gpu -#else -namespace cpu { -static Active::forward forward[] = FLOAT_ACTIVE_FUNCTION; -static Active::backward backward[] = FLOAT_ACTIVE_FUNCTION; - -static Active::forward forward_d[] = DOUBLE_ACTIVE_FUNCTION; -static Active::backward backward_d[] = DOUBLE_ACTIVE_FUNCTION; - -template -struct ForwardAct { - typename ForwardActType::type operator()(activation_mode_t type); -}; - -template <> -struct ForwardAct { - ForwardActType::type operator()(activation_mode_t type) { - return forward[type]; - } -}; - -template <> -struct ForwardAct { - ForwardActType::type operator()(activation_mode_t type) { - return forward_d[type]; - } -}; - -template -struct BackwardAct { - typename BackwardActType::type operator()(activation_mode_t type); -}; - -template <> -struct BackwardAct { - BackwardActType::type operator()(activation_mode_t type) { - return backward[type]; - } -}; - -template <> -struct BackwardAct { - BackwardActType::type operator()(activation_mode_t type) { - return backward_d[type]; - } -}; - -} // namespace cpu - -#ifdef __AVX__ -namespace avx { -static Active<__m256>::forward forward[] = AVX_ACTIVE_FUNCTION; -static Active<__m256>::backward backward[] = AVX_ACTIVE_FUNCTION; -} // namespace avx -#endif -#endif - -} // namespace hppl - -#endif // HL_ACTIVATION_FUNCTIONS_H_ diff --git a/paddle/operators/math/detail/hl_cpu_functions.cc b/paddle/operators/math/detail/hl_cpu_functions.cc deleted file mode 100644 index 21ec78f9629af0e4673a56517d76ac6734f57db8..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_cpu_functions.cc +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include -#include "hl_functions.h" - -namespace hppl { -namespace typef { - -float relu(const float a) { - return a > static_cast(0.0) ? a : static_cast(0.0); -} - -float sigmoid(const float a) { - const float min = SIGMOID_THRESHOLD_MIN; - const float max = SIGMOID_THRESHOLD_MAX; - float tmp = (a < min) ? min : ((a > max) ? max : a); - return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); -} - -float tanh(const float a) { - float tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(tmp))) - 1.0; -} - -float linear(const float a) { return a; } - -float relu(const float a, const float b) { return a * (b > 0.0 ? 1.0 : 0.0); } - -float sigmoid(const float a, const float b) { - return a * b * (static_cast(1) - b); -} - -float tanh(const float a, const float b) { - return a * (static_cast(1) - b * b); -} - -float linear(const float a, const float b) { return a; } - -} // namespace typef - -namespace typed { -double relu(const double a) { - return a > static_cast(0.0) ? a : static_cast(0.0); -} - -double sigmoid(const double a) { - const double min = SIGMOID_THRESHOLD_MIN; - const double max = SIGMOID_THRESHOLD_MAX; - double tmp = (a < min) ? min : ((a > max) ? max : a); - return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); -} - -double tanh(const double a) { - double tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(tmp))) - 1.0; -} - -double linear(const double a) { return a; } - -double relu(const double a, const double b) { - return a * (b > 0.0 ? 1.0 : 0.0); -} - -double sigmoid(const double a, const double b) { - return a * b * (static_cast(1) - b); -} - -double tanh(const double a, const double b) { - return a * (static_cast(1) - b * b); -} - -double linear(const double a, const double b) { return a; } - -} // namespace typed -} // namespace hppl diff --git a/paddle/operators/math/detail/hl_functions.h b/paddle/operators/math/detail/hl_functions.h deleted file mode 100644 index 3e2f0c9ee6d3ae2ed598c4d5f09b85b7d61fdd51..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_functions.h +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_FUNCTIONS_H_ -#define HL_FUNCTIONS_H_ - -/** - * sigmoid threshold maximum - */ -#define SIGMOID_THRESHOLD_MIN -40.0 - -/** - * sigmoid threshold minimum - */ -#define SIGMOID_THRESHOLD_MAX 13.0 - -/** - * The maximum input value for exp, used to avoid overflow problem. - * currently only used for tanh function. - */ -#define EXP_MAX_INPUT 40.0 - -#ifndef __NVCC__ -namespace hppl { -namespace typef { -float relu(const float a); -float sigmoid(const float a); -float tanh(const float a); -float linear(const float a); - -float relu(const float a, const float b); -float sigmoid(const float a, const float b); -float tanh(const float a, const float b); -float linear(const float a, const float b); - -} // namespace typef - -namespace typed { -double relu(const double a); -double sigmoid(const double a); -double tanh(const double a); -double linear(const double a); - -double relu(const double a, const double b); -double sigmoid(const double a, const double b); -double tanh(const double a, const double b); -double linear(const double a, const double b); -} // namespace typed - -} // namespace hppl - -#ifdef __AVX__ -#include "hl_avx_functions.h" -#endif - -#else -#include "hl_gpu_functions.h" -#endif - -#endif // HL_FUNCTIONS_H_ diff --git a/paddle/operators/math/detail/hl_gpu_functions.h b/paddle/operators/math/detail/hl_gpu_functions.h deleted file mode 100644 index 72f2204e7b2cfdba1367b51e3731dde11fb292d6..0000000000000000000000000000000000000000 --- a/paddle/operators/math/detail/hl_gpu_functions.h +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifndef HL_GPU_FUNCTIONS_CUH_ -#define HL_GPU_FUNCTIONS_CUH_ - -#include "hl_base.h" - -namespace hppl { -namespace typef { - -__device__ static float relu(const float a) { return a > 0.0f ? a : 0.0f; } - -__device__ static float sigmoid(const float a) { - const float min = SIGMOID_THRESHOLD_MIN; - const float max = SIGMOID_THRESHOLD_MAX; - float tmp = (a < min) ? min : ((a > max) ? max : a); - return __fdividef(1.0f, 1.0f + __expf(-tmp)); -} - -__device__ static float tanh(const float a) { - float tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return __fdividef(2.0f, (1.0f + __expf(-2.0f * tmp))) - 1.0f; -} - -__device__ static float linear(const float a) { return a; } - -__device__ static float relu(const float a, const float b) { - return a * (b > 0.0f ? 1.0f : 0.0f); -} - -__device__ static float sigmoid(const float a, const float b) { - return a * b * (1.0f - b); -} - -__device__ static float tanh(const float a, const float b) { - return a * (1.0f - b * b); -} - -__device__ static float linear(const float a, const float b) { return a; } - -} // namespace typef - -namespace typed { - -__device__ static double relu(const double a) { return a > 0.0 ? a : 0.0; } - -__device__ static double sigmoid(const double a) { - const double min = SIGMOID_THRESHOLD_MIN; - const double max = SIGMOID_THRESHOLD_MAX; - double tmp = (a < min) ? min : ((a > max) ? max : a); - return 1.0 / (1.0 + exp(-tmp)); -} - -__device__ static double tanh(const double a) { - double tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(-2.0 * a))) - 1.0; -} - -__device__ static double linear(const double a) { return a; } - -__device__ static double relu(const double a, const double b) { - return a * (b > 0.0 ? 1.0 : 0.0); -} - -__device__ static double sigmoid(const double a, const double b) { - return a * b * (1 - b); -} - -__device__ static double tanh(const double a, const double b) { - return a * (1.0 - b * b); -} - -__device__ static double linear(const double a, const double b) { return a; } - -} // namespace typef - -} // namespace hppl - -#endif // HL_GPU_FUNCTIONS_CUH_ diff --git a/paddle/operators/math/detail/lstm_cpu_kernel.h b/paddle/operators/math/detail/lstm_cpu_kernel.h index d0ed55ea168bc3e701c421c51d662c646e475351..fc3ad0ce58aa1552ef7e717fb529c2d454b4895a 100644 --- a/paddle/operators/math/detail/lstm_cpu_kernel.h +++ b/paddle/operators/math/detail/lstm_cpu_kernel.h @@ -14,7 +14,7 @@ limitations under the License. */ #pragma once #include -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/operators/math/lstm_compute.h" namespace paddle { @@ -26,7 +26,10 @@ namespace detail { template void naive_lstm_forward_one_sequence(Op op, LstmMetaValue value, - int frameSize) { + int frameSize, + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { T rValueIn; T rValueIg; T rValueFg; @@ -49,16 +52,16 @@ void naive_lstm_forward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = value.checkIg[i]; - rCheckF = value.checkFg[i]; - rCheckO = value.checkOg[i]; + rCheckI = value.checkIg ? value.checkIg[i] : 0; + rCheckF = value.checkFg ? value.checkFg[i] : 0; + rCheckO = value.checkOg ? value.checkOg[i] : 0; if (value.prevStateValue) { rPrevState = value.prevStateValue[i]; } op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); valueIn[i] = rValueIn; valueIg[i] = rValueIg; @@ -72,7 +75,10 @@ void naive_lstm_forward_one_sequence(Op op, LstmMetaValue value, template void naive_lstm_backward_one_sequence(Op op, LstmMetaValue value, - LstmMetaGrad grad, int frameSize) { + LstmMetaGrad grad, int frameSize, + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { T rValueIn; T rValueIg; T rValueFg; @@ -108,9 +114,9 @@ void naive_lstm_backward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = value.checkIg[i]; - rCheckF = value.checkFg[i]; - rCheckO = value.checkOg[i]; + rCheckI = value.checkIg ? value.checkIg[i] : 0; + rCheckF = value.checkFg ? value.checkFg[i] : 0; + rCheckO = value.checkOg ? value.checkOg[i] : 0; rState = value.stateValue[i]; rStateAtv = value.stateActiveValue[i]; rOutputGrad = grad.outputGrad[i]; @@ -122,7 +128,7 @@ void naive_lstm_backward_one_sequence(Op op, LstmMetaValue value, op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, - rCheckOGrad); + rCheckOGrad, active_node, active_gate, active_state); gradIn[i] = rGradIn; gradIg[i] = rGradIg; @@ -149,9 +155,9 @@ void avx_lstm_forward_one_sequence(Op op, LstmMetaValue value, int frameSize, __m256 rValueIg; __m256 rValueFg; __m256 rValueOg; - __m256 rCheckI; - __m256 rCheckF; - __m256 rCheckO; + __m256 rCheckI = _mm256_set1_ps(0.0f); + __m256 rCheckF = _mm256_set1_ps(0.0f); + __m256 rCheckO = _mm256_set1_ps(0.0f); __m256 rState; __m256 rPrevState = _mm256_set1_ps(0.0f); __m256 rStateAtv; @@ -167,17 +173,18 @@ void avx_lstm_forward_one_sequence(Op op, LstmMetaValue value, int frameSize, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = ((__m256 *)value.checkIg)[i]; - rCheckF = ((__m256 *)value.checkFg)[i]; - rCheckO = ((__m256 *)value.checkOg)[i]; + if (value.checkIg) { + rCheckI = ((__m256 *)value.checkIg)[i]; + rCheckF = ((__m256 *)value.checkFg)[i]; + rCheckO = ((__m256 *)value.checkOg)[i]; + } if (value.prevStateValue) { rPrevState = ((__m256 *)value.prevStateValue)[i]; } op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO, hppl::avx::forward[active_node], - hppl::avx::forward[active_gate], hppl::avx::forward[active_state]); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); valueIn[i] = rValueIn; valueIg[i] = rValueIg; @@ -211,9 +218,9 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, __m256 rState; __m256 rStateAtv; __m256 rOutputGrad; - __m256 rCheckI; - __m256 rCheckF; - __m256 rCheckO; + __m256 rCheckI = _mm256_set1_ps(0.0f); + __m256 rCheckF = _mm256_set1_ps(0.0f); + __m256 rCheckO = _mm256_set1_ps(0.0f); __m256 rCheckIGrad; __m256 rCheckFGrad; __m256 rCheckOGrad; @@ -232,9 +239,11 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, rValueIg = valueIg[i]; rValueFg = valueFg[i]; rValueOg = valueOg[i]; - rCheckI = ((__m256 *)value.checkIg)[i]; - rCheckF = ((__m256 *)value.checkFg)[i]; - rCheckO = ((__m256 *)value.checkOg)[i]; + if (value.checkIg) { + rCheckI = ((__m256 *)value.checkIg)[i]; + rCheckF = ((__m256 *)value.checkFg)[i]; + rCheckO = ((__m256 *)value.checkOg)[i]; + } rState = ((__m256 *)value.stateValue)[i]; rStateAtv = ((__m256 *)value.stateActiveValue)[i]; rOutputGrad = ((__m256 *)grad.outputGrad)[i]; @@ -246,8 +255,7 @@ void avx_lstm_backward_one_sequence(Op op, LstmMetaValue value, op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, - rCheckOGrad, hppl::avx::backward[active_node], - hppl::avx::backward[active_gate], hppl::avx::backward[active_state]); + rCheckOGrad, active_node, active_gate, active_state); gradIn[i] = rGradIn; gradIg[i] = rGradIg; @@ -274,7 +282,8 @@ void cpu_lstm_forward(Op op, LstmMetaValue value, int frameSize, avx_lstm_forward_one_sequence(op, value, frameSize, active_node, active_gate, active_state); } else { - naive_lstm_forward_one_sequence(op, value, frameSize); + naive_lstm_forward_one_sequence(op, value, frameSize, active_node, + active_gate, active_state); } } @@ -287,7 +296,8 @@ void cpu_lstm_backward(Op op, LstmMetaValue value, LstmMetaGrad grad, avx_lstm_backward_one_sequence(op, value, grad, frameSize, active_node, active_gate, active_state); } else { - naive_lstm_backward_one_sequence(op, value, grad, frameSize); + naive_lstm_backward_one_sequence(op, value, grad, frameSize, active_node, + active_gate, active_state); } } diff --git a/paddle/operators/math/detail/lstm_gpu_kernel.h b/paddle/operators/math/detail/lstm_gpu_kernel.h index c06f164f84a92d31f89901e2656bdb8e69c533b7..d138bbe411f69929a14ad19af3e84824ac7a5d58 100644 --- a/paddle/operators/math/detail/lstm_gpu_kernel.h +++ b/paddle/operators/math/detail/lstm_gpu_kernel.h @@ -13,13 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/operators/math/lstm_compute.h" #include "paddle/platform/cuda_helper.h" #include "paddle/platform/device_context.h" -#include +#include namespace paddle { namespace operators { @@ -32,7 +31,9 @@ namespace detail { */ template __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, - int batchSize) { + int batchSize, activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; if (frameIdx >= frameSize) return; @@ -54,9 +55,10 @@ __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, T rValueIg; T rValueFg; T rValueOg; - T rCheckI = value.checkIg[frameIdx]; - T rCheckF = value.checkFg[frameIdx]; - T rCheckO = value.checkOg[frameIdx]; + + T rCheckI = value.checkIg ? value.checkIg[frameIdx] : 0; + T rCheckF = value.checkFg ? value.checkFg[frameIdx] : 0; + T rCheckO = value.checkOg ? value.checkOg[frameIdx] : 0; rValueIn = value.gateValue[frameIdx]; rValueIg = value.gateValue[frameIdx + frameSize]; @@ -69,7 +71,7 @@ __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, } op(rValueIn, rValueIg, rValueFg, rValueOg, rPrevState, rState, rStateAtv, - rOut, rCheckI, rCheckF, rCheckO); + rOut, rCheckI, rCheckF, rCheckO, active_node, active_gate, active_state); value.gateValue[frameIdx] = rValueIn; value.gateValue[frameIdx + frameSize] = rValueIg; @@ -88,7 +90,9 @@ __global__ void KeLstmForward(Op op, LstmMetaValue value, int frameSize, template __global__ void KeLstmBackward(Op op, LstmMetaValue value, LstmMetaGrad grad, int frameSize, - int batchSize) { + int batchSize, activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { const int frameIdx = blockIdx.x * blockDim.x + threadIdx.x; if (frameIdx >= frameSize) return; @@ -118,9 +122,10 @@ __global__ void KeLstmBackward(Op op, LstmMetaValue value, T rStateGrad; T rStateAtv; T rOutputGrad; - T rCheckI = value.checkIg[frameIdx]; - T rCheckF = value.checkFg[frameIdx]; - T rCheckO = value.checkOg[frameIdx]; + T rCheckI = value.checkIg ? value.checkIg[frameIdx] : 0; + T rCheckF = value.checkFg ? value.checkFg[frameIdx] : 0; + T rCheckO = value.checkOg ? value.checkOg[frameIdx] : 0; + T rCheckIGrad; T rCheckFGrad; T rCheckOGrad; @@ -141,7 +146,8 @@ __global__ void KeLstmBackward(Op op, LstmMetaValue value, op(rValueIn, rValueIg, rValueFg, rValueOg, rGradIn, rGradIg, rGradFg, rGradOg, rPrevState, rPrevStateGrad, rState, rStateGrad, rStateAtv, rOutputGrad, - rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, rCheckOGrad); + rCheckI, rCheckF, rCheckO, rCheckIGrad, rCheckFGrad, rCheckOGrad, + active_node, active_gate, active_state); grad.gateGrad[frameIdx] = rGradIn; grad.gateGrad[frameIdx + frameSize] = rGradIg; @@ -197,11 +203,13 @@ void gpu_lstm_forward(const platform::DeviceContext& context, Op op, if (batchSize == 1) { KeLstmForward<<>>( - op, value, frameSize, batchSize); + op, value, frameSize, batchSize, active_node, active_gate, + active_state); } else { KeLstmForward<<>>( - op, value, frameSize, batchSize); + op, value, frameSize, batchSize, active_node, active_gate, + active_state); } } @@ -220,9 +228,9 @@ void gpu_lstm_backward(const platform::DeviceContext& context, Op op, threads = dim3(framePerBlock, 1); grid = dim3(frameBlocks, 1); } else { - /* framePerBlock = 32 batchPerBlock = 32 */ - threads = dim3(32, 32); - grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 32 - 1) / 32); + /* framePerBlock = 32 batchPerBlock = 16 */ + threads = dim3(32, 16); + grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 16 - 1) / 16); } auto stream = @@ -230,11 +238,13 @@ void gpu_lstm_backward(const platform::DeviceContext& context, Op op, if (batchSize == 1) { KeLstmBackward<<>>( - op, value, grad, frameSize, batchSize); + op, value, grad, frameSize, batchSize, active_node, active_gate, + active_state); } else { KeLstmBackward<<>>( - op, value, grad, frameSize, batchSize); + op, value, grad, frameSize, batchSize, active_node, active_gate, + active_state); } } diff --git a/paddle/operators/math/detail/lstm_kernel.h b/paddle/operators/math/detail/lstm_kernel.h index 461039a4d51a2b9b8a55d3101bdf4c511907597e..9daaf91981a8e0252374f528f0e063111bd32675 100644 --- a/paddle/operators/math/detail/lstm_kernel.h +++ b/paddle/operators/math/detail/lstm_kernel.h @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/math/detail/hl_activation_functions.h" +#include "paddle/operators/math/detail/activation_functions.h" #include "paddle/platform/hostdevice.h" #include @@ -24,45 +24,22 @@ namespace detail { namespace forward { -template -DEVICE inline T sigmoid(const T a) { - const T min = SIGMOID_THRESHOLD_MIN; - const T max = SIGMOID_THRESHOLD_MAX; - T tmp = (a < min) ? min : ((a > max) ? max : a); - return static_cast(1.0) / (static_cast(1.0) + exp(-tmp)); -} - -template -DEVICE inline T tanh(const T a) { - T tmp = -2.0 * a; - tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; - return (2.0 / (1.0 + exp(tmp))) - 1.0; -} - template class lstm { public: HOSTDEVICE void operator()(T &valueIn, T &valueIg, T &valueFg, T &valueOg, T &prevState, T &state, T &stateAtv, T &output, - T &checkI, T &checkF, T &checkO) { -#if 0 - // TODO(qingqing) support to activation speficed by users - valueIn = actInput(valueIn); - valueIg = actGate(valueIg + prevState * checkI); - valueFg = actGate(valueFg + prevState * checkF); - state = valueIn * valueIg + prevState * valueFg; - valueOg = actGate(valueOg + state * checkO); - stateAtv = actState(state); - output = valueOg * stateAtv; -#else - valueIn = tanh(valueIn); - valueIg = sigmoid(valueIg + prevState * checkI); - valueFg = sigmoid(valueFg + prevState * checkF); + T &checkI, T &checkF, T &checkO, + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + valueIn = activation(valueIn, active_node); + valueIg = activation(valueIg + prevState * checkI, active_gate); + valueFg = activation(valueFg + prevState * checkF, active_gate); state = valueIn * valueIg + prevState * valueFg; - valueOg = sigmoid(valueOg + state * checkO); - stateAtv = tanh(state); + valueOg = activation(valueOg + state * checkO, active_gate); + stateAtv = activation(state, active_state); output = valueOg * stateAtv; -#endif } #ifndef __NVCC__ #ifndef __AVX__ // If not compiled with AVX instructs. Disable AVX by default @@ -75,16 +52,19 @@ class lstm { __m256 &valueOg, __m256 &prevState, __m256 &state, __m256 &stateAtv, __m256 &output, __m256 &checkI, __m256 &checkF, __m256 &checkO, - hppl::Active<__m256>::forward actInput, - hppl::Active<__m256>::forward actGate, - hppl::Active<__m256>::forward actState) { - valueIn = actInput(valueIn); - valueIg = actGate(_mm256_add_ps(valueIg, _mm256_mul_ps(prevState, checkI))); - valueFg = actGate(_mm256_add_ps(valueFg, _mm256_mul_ps(prevState, checkF))); + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + valueIn = activation(valueIn, active_node); + valueIg = activation( + _mm256_add_ps(valueIg, _mm256_mul_ps(prevState, checkI)), active_gate); + valueFg = activation( + _mm256_add_ps(valueFg, _mm256_mul_ps(prevState, checkF)), active_gate); state = _mm256_add_ps(_mm256_mul_ps(valueIn, valueIg), _mm256_mul_ps(prevState, valueFg)); - valueOg = actGate(_mm256_add_ps(valueOg, _mm256_mul_ps(state, checkO))); - stateAtv = actState(state); + valueOg = activation(_mm256_add_ps(valueOg, _mm256_mul_ps(state, checkO)), + active_gate); + stateAtv = activation(state, active_state); output = _mm256_mul_ps(valueOg, stateAtv); } #endif @@ -95,16 +75,6 @@ class lstm { namespace backward { -template -DEVICE inline T sigmoid(const T a, const T b) { - return a * b * (1.0 - b); -} - -template -DEVICE inline T tanh(const T a, const T b) { - return a * (1.0 - b * b); -} - template class lstm { public: @@ -113,29 +83,20 @@ class lstm { T &prevState, T &prevStateGrad, T &state, T &stateGrad, T &stateAtv, T &outputGrad, T &checkI, T &checkF, T &checkO, T &checkIGrad, - T &checkFGrad, T &checkOGrad) { -#if 0 - // TODO(qingqing) support to activation speficed by users - gradOg = actGate(outputGrad * stateAtv, valueOg); - stateGrad += actState(outputGrad * valueOg, stateAtv) + gradOg * checkO; - gradIn = actInput(stateGrad * valueIg, valueIn); - gradIg = actGate(stateGrad * valueIn, valueIg); - gradFg = actGate(stateGrad * prevState, valueFg); + T &checkFGrad, T &checkOGrad, + activation_mode_t active_node, + activation_mode_t active_gate, + activation_mode_t active_state) { + gradOg = activation(outputGrad * stateAtv, valueOg, active_gate); + stateGrad += activation(outputGrad * valueOg, stateAtv, active_state) + + gradOg * checkO; + gradIn = activation(stateGrad * valueIg, valueIn, active_node); + gradIg = activation(stateGrad * valueIn, valueIg, active_gate); + gradFg = activation(stateGrad * prevState, valueFg, active_gate); prevStateGrad = gradIg * checkI + gradFg * checkF + stateGrad * valueFg; checkIGrad = gradIg * prevState; checkFGrad = gradFg * prevState; checkOGrad = gradOg * state; -#else - gradOg = sigmoid(outputGrad * stateAtv, valueOg); - stateGrad += tanh(outputGrad * valueOg, stateAtv) + gradOg * checkO; - gradIn = tanh(stateGrad * valueIg, valueIn); - gradIg = sigmoid(stateGrad * valueIn, valueIg); - gradFg = sigmoid(stateGrad * prevState, valueFg); - prevStateGrad = gradIg * checkI + gradFg * checkF + stateGrad * valueFg; - checkIGrad = gradIg * prevState; - checkFGrad = gradFg * prevState; - checkOGrad = gradOg * state; -#endif } #ifndef __NVCC__ #ifndef __AVX__ // If not compiled with AVX instructs. Disable AVX by default @@ -143,24 +104,26 @@ class lstm { #else // Only float support AVX optimization static const bool avx = std::is_same::value; - HOSTDEVICE void operator()(__m256 &valueIn, __m256 &valueIg, __m256 &valueFg, - __m256 &valueOg, __m256 &gradIn, __m256 &gradIg, - __m256 &gradFg, __m256 &gradOg, __m256 &prevState, - __m256 &prevStateGrad, __m256 &state, - __m256 &stateGrad, __m256 &stateAtv, - __m256 &outputGrad, __m256 &checkI, __m256 &checkF, - __m256 &checkO, __m256 &checkIGrad, - __m256 &checkFGrad, __m256 &checkOGrad, - hppl::Active<__m256>::backward actInput, - hppl::Active<__m256>::backward actGate, - hppl::Active<__m256>::backward actState) { - gradOg = actGate(_mm256_mul_ps(outputGrad, stateAtv), valueOg); + HOSTDEVICE void operator()( + __m256 &valueIn, __m256 &valueIg, __m256 &valueFg, __m256 &valueOg, + __m256 &gradIn, __m256 &gradIg, __m256 &gradFg, __m256 &gradOg, + __m256 &prevState, __m256 &prevStateGrad, __m256 &state, + __m256 &stateGrad, __m256 &stateAtv, __m256 &outputGrad, __m256 &checkI, + __m256 &checkF, __m256 &checkO, __m256 &checkIGrad, __m256 &checkFGrad, + __m256 &checkOGrad, activation_mode_t active_node, + activation_mode_t active_gate, activation_mode_t active_state) { + gradOg = + activation(_mm256_mul_ps(outputGrad, stateAtv), valueOg, active_gate); stateGrad = _mm256_add_ps( - actState(_mm256_mul_ps(outputGrad, valueOg), stateAtv), stateGrad); + activation(_mm256_mul_ps(outputGrad, valueOg), stateAtv, active_state), + stateGrad); stateGrad = _mm256_add_ps(_mm256_mul_ps(gradOg, checkO), stateGrad); - gradIn = actInput(_mm256_mul_ps(stateGrad, valueIg), valueIn); - gradIg = actGate(_mm256_mul_ps(stateGrad, valueIn), valueIg); - gradFg = actGate(_mm256_mul_ps(stateGrad, prevState), valueFg); + gradIn = + activation(_mm256_mul_ps(stateGrad, valueIg), valueIn, active_node); + gradIg = + activation(_mm256_mul_ps(stateGrad, valueIn), valueIg, active_gate); + gradFg = + activation(_mm256_mul_ps(stateGrad, prevState), valueFg, active_gate); prevStateGrad = _mm256_add_ps(_mm256_mul_ps(gradIg, checkI), _mm256_mul_ps(gradFg, checkF)); prevStateGrad = diff --git a/paddle/operators/math/gru_compute.cc b/paddle/operators/math/gru_compute.cc new file mode 100644 index 0000000000000000000000000000000000000000..125af449d3f700e24be5e4b7615c3b0e03fd4e5b --- /dev/null +++ b/paddle/operators/math/gru_compute.cc @@ -0,0 +1,102 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/gru_compute.h" +#include "paddle/operators/math/detail/gru_cpu_kernel.h" +#include "paddle/operators/math/detail/gru_kernel.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { +namespace math { + +template +struct GRUUnitFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, int frameSize, int batchSize, + activation_mode_t active_node, + activation_mode_t active_gate) { +#ifndef __NVCC__ + if (value.prevOutValue) { + math::gemm( + context, false, false, batchSize, frameSize * 2, frameSize, 1, + value.prevOutValue, frameSize, value.gateWeight, frameSize * 2, 1, + value.gateValue, frameSize * 3); + } + + detail::forward_reset_output(detail::forward::gru_resetOutput(), value, + frameSize, batchSize, active_gate); + + if (value.prevOutValue) { + math::gemm( + context, false, false, batchSize, frameSize, frameSize, 1, + value.resetOutputValue, frameSize, value.stateWeight, frameSize, 1, + value.gateValue + frameSize * 2, frameSize * 3); + } + + detail::forward_final_output(detail::forward::gru_finalOutput(), value, + frameSize, batchSize, active_node); +#endif + } +}; + +template +struct GRUUnitGradFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, hl_gru_grad grad, int frameSize, + int batchSize, activation_mode_t active_node, + activation_mode_t active_gate) { +#ifndef __NVCC__ + detail::backward_state_grad(detail::backward::gru_stateGrad(), value, + grad, frameSize, batchSize, active_node); + + if (value.prevOutValue && grad.prevOutGrad) { + math::gemm( + context, false, true, batchSize, frameSize, frameSize, 1, + grad.gateGrad + frameSize * 2, frameSize * 3, value.stateWeight, + frameSize, 0, grad.resetOutputGrad, frameSize); + + if (grad.stateWeightGrad) { + math::gemm( + context, true, false, frameSize, frameSize, batchSize, 1, + value.resetOutputValue, frameSize, grad.gateGrad + frameSize * 2, + frameSize * 3, 1, grad.stateWeightGrad, frameSize); + } + } + + detail::backward_reset_grad(detail::backward::gru_resetGrad(), value, + grad, frameSize, batchSize, active_gate); + + if (grad.prevOutGrad && value.prevOutValue) { + math::gemm( + context, false, true, batchSize, frameSize, frameSize * 2, 1, + grad.gateGrad, frameSize * 3, value.gateWeight, frameSize * 2, 1, + grad.prevOutGrad, frameSize); + + if (grad.gateWeightGrad) { + math::gemm( + context, true, false, frameSize, frameSize * 2, batchSize, 1, + value.prevOutValue, frameSize, grad.gateGrad, frameSize * 3, 1, + grad.gateWeightGrad, frameSize * 2); + } + } +#endif + } +}; + +template struct GRUUnitFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitGradFunctor; +template struct GRUUnitGradFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/gru_compute.cu b/paddle/operators/math/gru_compute.cu new file mode 100644 index 0000000000000000000000000000000000000000..7b9e54ac029f6aa00553338435684097d6d02b25 --- /dev/null +++ b/paddle/operators/math/gru_compute.cu @@ -0,0 +1,178 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/detail/gru_gpu_kernel.h" +#include "paddle/operators/math/detail/gru_kernel.h" +#include "paddle/operators/math/gru_compute.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { +namespace math { + +template +struct GRUUnitFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, int frameSize, int batchSize, + activation_mode_t active_node, + activation_mode_t active_gate) { + auto stream = + reinterpret_cast(context).stream(); + dim3 threads; + dim3 grid; + if (batchSize == 1) { + int framePerBlock = frameSize <= 1024 ? frameSize : 1024; + int frameBlocks = (frameSize + 1024 - 1) / 1024; + threads = dim3(framePerBlock, 1); + grid = dim3(frameBlocks, 1); + } else { + threads = dim3(32, 32); + grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 32 - 1) / 32); + } + + if (value.prevOutValue) { + math::gemm( + context, false, false, batchSize, frameSize * 2, frameSize, 1, + value.prevOutValue, frameSize, value.gateWeight, frameSize * 2, 1, + value.gateValue, frameSize * 3); + } + + if (batchSize == 1) { + detail::KeGruForwardResetOutput, + /* isBatch= */ false, + T><<>>( + detail::forward::gru_resetOutput(), value.gateValue, + value.resetOutputValue, value.prevOutValue, frameSize, batchSize, + active_gate); + } else { + detail::KeGruForwardResetOutput, + /* isBatch= */ true, + T><<>>( + detail::forward::gru_resetOutput(), value.gateValue, + value.resetOutputValue, value.prevOutValue, frameSize, batchSize, + active_gate); + } + + if (value.prevOutValue) { + math::gemm( + context, false, false, batchSize, frameSize, frameSize, 1, + value.resetOutputValue, frameSize, value.stateWeight, frameSize, 1, + value.gateValue + frameSize * 2, frameSize * 3); + } + + if (batchSize == 1) { + detail::KeGruForwardFinalOutput, + /* isBatch= */ false, + T><<>>( + detail::forward::gru_finalOutput(), value.gateValue, + value.prevOutValue, value.outputValue, frameSize, batchSize, + active_node); + } else { + detail::KeGruForwardFinalOutput, + /* isBatch= */ true, + T><<>>( + detail::forward::gru_finalOutput(), value.gateValue, + value.prevOutValue, value.outputValue, frameSize, batchSize, + active_node); + } + } +}; + +template +struct GRUUnitGradFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, hl_gru_grad grad, int frameSize, + int batchSize, activation_mode_t active_node, + activation_mode_t active_gate) { + auto stream = + reinterpret_cast(context).stream(); + dim3 threads; + dim3 grid; + if (batchSize == 1) { + int framePerBlock = frameSize <= 1024 ? frameSize : 1024; + int frameBlocks = (frameSize + 1024 - 1) / 1024; + threads = dim3(framePerBlock, 1); + grid = dim3(frameBlocks, 1); + } else { + threads = dim3(32, 32); + grid = dim3((frameSize + 32 - 1) / 32, (batchSize + 32 - 1) / 32); + } + + if (batchSize == 1) { + detail::KeGruBackwardStateGrad< + detail::backward::gru_stateGrad, + /* isBatch= */ false><<>>( + detail::backward::gru_stateGrad(), value.gateValue, grad.gateGrad, + value.prevOutValue, grad.prevOutGrad, grad.outputGrad, frameSize, + batchSize, active_node); + } else { + detail::KeGruBackwardStateGrad< + detail::backward::gru_stateGrad, + /* isBatch= */ true><<>>( + detail::backward::gru_stateGrad(), value.gateValue, grad.gateGrad, + value.prevOutValue, grad.prevOutGrad, grad.outputGrad, frameSize, + batchSize, active_node); + } + + if (value.prevOutValue && grad.prevOutGrad) { + math::gemm( + context, false, true, batchSize, frameSize, frameSize, 1, + grad.gateGrad + frameSize * 2, frameSize * 3, value.stateWeight, + frameSize, 0, grad.resetOutputGrad, frameSize); + + if (grad.stateWeightGrad) { + math::gemm( + context, true, false, frameSize, frameSize, batchSize, 1, + value.resetOutputValue, frameSize, grad.gateGrad + frameSize * 2, + frameSize * 3, 1, grad.stateWeightGrad, frameSize); + } + } + + if (batchSize == 1) { + detail::KeGruBackwardResetGrad< + detail::backward::gru_resetGrad, + /* isBatch= */ false><<>>( + detail::backward::gru_resetGrad(), value.gateValue, grad.gateGrad, + value.prevOutValue, grad.prevOutGrad, grad.resetOutputGrad, frameSize, + batchSize, active_gate); + } else { + detail::KeGruBackwardResetGrad< + detail::backward::gru_resetGrad, + /* isBatch= */ true><<>>( + detail::backward::gru_resetGrad(), value.gateValue, grad.gateGrad, + value.prevOutValue, grad.prevOutGrad, grad.resetOutputGrad, frameSize, + batchSize, active_gate); + } + + if (grad.prevOutGrad && value.prevOutValue) { + math::gemm( + context, false, true, batchSize, frameSize, frameSize * 2, 1, + grad.gateGrad, frameSize * 3, value.gateWeight, frameSize * 2, 1, + grad.prevOutGrad, frameSize); + + if (grad.gateWeightGrad) { + math::gemm( + context, true, false, frameSize, frameSize * 2, batchSize, 1, + value.prevOutValue, frameSize, grad.gateGrad, frameSize * 3, 1, + grad.gateWeightGrad, frameSize * 2); + } + } + } +}; + +template struct GRUUnitFunctor; +template struct GRUUnitFunctor; +template struct GRUUnitGradFunctor; +template struct GRUUnitGradFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/gru_compute.h b/paddle/operators/math/gru_compute.h new file mode 100644 index 0000000000000000000000000000000000000000..1475fb38104f353857dfd968e46af98a6d52c52a --- /dev/null +++ b/paddle/operators/math/gru_compute.h @@ -0,0 +1,61 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/operators/math/lstm_compute.h" +#include "paddle/platform/device_context.h" +#include "paddle/platform/enforce.h" + +namespace paddle { +namespace operators { +namespace math { + +// TODO(guosheng): refine code style in gru_compute +template +struct hl_gru_value { + T *gateWeight; + T *stateWeight; + T *gateValue; + T *resetOutputValue; + T *outputValue; + T *prevOutValue; +}; + +template +struct hl_gru_grad { + T *gateWeightGrad; + T *stateWeightGrad; + T *gateGrad; + T *resetOutputGrad; + T *outputGrad; + T *prevOutGrad; +}; + +template +struct GRUUnitFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, int frameSize, int batchSize, + activation_mode_t active_node, + activation_mode_t active_gate); +}; + +template +struct GRUUnitGradFunctor { + static void compute(const platform::DeviceContext &context, + hl_gru_value value, hl_gru_grad grad, int frameSize, + int batchSize, activation_mode_t active_node, + activation_mode_t active_gate); +}; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index 2a9c09a0f16b71473e21765ab9253eb7b8bcf28c..1b0d4c8bdc683b5203a4bc4b3838560cffe00bc8 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/operators/math/math_function.h" +#include "paddle/framework/data_type.h" namespace paddle { namespace operators { @@ -233,6 +234,52 @@ void gemv(const platform::DeviceContext& context, template struct SetConstant; +struct TensorSetConstantCPU { + TensorSetConstantCPU(framework::Tensor* tensor, float value) + : tensor_(tensor), value_(value) {} + template + void operator()() const { + auto cpu = platform::CPUPlace(); + auto* begin = tensor_->mutable_data(cpu); + std::fill(begin, begin + tensor_->numel(), static_cast(value_)); + } + framework::Tensor* tensor_; + float value_; +}; + +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + framework::VisitDataType(framework::ToDataType(tensor->type()), + TensorSetConstantCPU(tensor, value)); +} + +struct TensorSetConstantWithPlace : public boost::static_visitor { + TensorSetConstantWithPlace(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) + : context_(context), tensor_(tensor), value_(value) {} + + template + void operator()(Place place) const { + set_constant_with_place(context_, tensor_, value_); + } + + const platform::DeviceContext& context_; + framework::Tensor* tensor_; + float value_; +}; + +void set_constant(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) { + TensorSetConstantWithPlace func(context, tensor, value); +#ifdef PADDLE_WITH_CUDA + tensor->place().apply_visitor(func); +#else + func(platform::CPUPlace()); +#endif +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index e6fd8bf235b8539702ca2c5b39e305cb1becf5cb..817deec94314bdfd2ed7e4b0ba5212c72b813455 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include "paddle/framework/data_type.h" #include "paddle/operators/math/math_function.h" namespace paddle { @@ -232,6 +233,30 @@ void gemv(const platform::DeviceContext& context, template struct SetConstant; +struct TensorSetConstantGPU { + TensorSetConstantGPU(const platform::DeviceContext& context, + framework::Tensor* tensor, float value) + : context_(context), tensor_(tensor), value_(value) {} + + template + void operator()() const { + SetConstant functor; + functor(context_, tensor_, static_cast(value_)); + } + + const platform::DeviceContext& context_; + framework::Tensor* tensor_; + float value_; +}; + +template <> +void set_constant_with_place( + const platform::DeviceContext& context, framework::Tensor* tensor, + float value) { + framework::VisitDataType(framework::ToDataType(tensor->type()), + TensorSetConstantGPU(context, tensor, value)); +} + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 3bb5aa0332c7e2a63d20b91893c03ccd468dd863..c2aaa1d7b7e920c3e6fd9ae4424eae725c3b7c0e 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -19,11 +19,6 @@ limitations under the License. */ #include #endif -#ifdef PADDLE_USE_MKL -#include -#include -#endif - #ifdef PADDLE_USE_ATLAS extern "C" { #include @@ -108,6 +103,13 @@ struct SetConstant { } }; +template +void set_constant_with_place(const platform::DeviceContext& context, + framework::Tensor* tensor, float value); + +void set_constant(const platform::DeviceContext& context, + framework::Tensor* tensor, float value); + } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 7d84ad9aadb2892db0d0ee9cab428dc5036614e9..983c9fdcffb0a67da1bc0b5b4af9420a68bd2ac1 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -139,3 +139,15 @@ TEST(math_function, gemv) { GemvTest(12, 7, true); GemvTest(7, 9, true); } + +TEST(math_funciton, set_constant) { + paddle::framework::Tensor t; + t.Resize({10, 10}); + t.mutable_data(paddle::platform::CPUPlace()); + auto* ctx = new paddle::platform::CPUDeviceContext(); + paddle::operators::math::set_constant(*ctx, &t, 10); + for (int64_t i = 0; i < t.numel(); ++i) { + PADDLE_ENFORCE_EQ(10, t.data()[i]); + } + delete ctx; +} diff --git a/paddle/operators/math/sequence2batch.cc b/paddle/operators/math/sequence2batch.cc index 10c6e105b950b9d510e7a14828d72531e8eb0028..5b3bde02fbf981772759caa3d0054fac4a8520f9 100644 --- a/paddle/operators/math/sequence2batch.cc +++ b/paddle/operators/math/sequence2batch.cc @@ -22,8 +22,8 @@ template class CopyMatrixRowsFunctor { public: void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index) { + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); auto dst_dims = dst.dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2UL, diff --git a/paddle/operators/math/sequence2batch.cu b/paddle/operators/math/sequence2batch.cu index 4f349946785171e6c59b22163ba76791c7244f88..8d04653832d58aa048f73e53b8349a08da3145a4 100644 --- a/paddle/operators/math/sequence2batch.cu +++ b/paddle/operators/math/sequence2batch.cu @@ -41,8 +41,8 @@ template class CopyMatrixRowsFunctor { public: void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index) { + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index) { auto src_dims = src.dims(); auto dst_dims = dst.dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2, diff --git a/paddle/operators/math/sequence2batch.h b/paddle/operators/math/sequence2batch.h index b1ba35a6d4a891e9152ac2088bc76e3969be6405..794c7d43973924d470124baf8c0c3de66e4ba087 100644 --- a/paddle/operators/math/sequence2batch.h +++ b/paddle/operators/math/sequence2batch.h @@ -30,8 +30,8 @@ class CopyMatrixRowsFunctor { // copy the input src to the indexed rows of output dst. // The indexed rows are based on the input index. void operator()(const platform::DeviceContext& context, - const framework::LoDTensor& src, const size_t* index, - framework::LoDTensor& dst, bool is_src_index); + const framework::Tensor& src, const size_t* index, + framework::Tensor& dst, bool is_src_index); }; template @@ -57,7 +57,7 @@ class LoDTensor2BatchFunctor { bool is_reverse = false) const { if (!is_cal_batch_lod) { auto lods = batch.lod(); - PADDLE_ENFORCE_EQ(lods.size(), 2UL); + PADDLE_ENFORCE_GT(lods.size(), 2UL); PADDLE_ENFORCE_EQ(lods[1].size(), static_cast(lod_tensor.dims()[0])); CopyMatrixRowsFunctor to_batch; @@ -66,8 +66,8 @@ class LoDTensor2BatchFunctor { } auto lods = lod_tensor.lod(); - PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); auto lod = lods[0]; + PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); std::vector seq_info; for (size_t seq_id = 0; seq_id < lod.size() - 1; ++seq_id) { @@ -78,8 +78,7 @@ class LoDTensor2BatchFunctor { std::sort(seq_info.begin(), seq_info.end(), [](SeqInfo a, SeqInfo b) { return a.length > b.length; }); - // calculate the start position of each batch - // (numBatch equal the maxLength of sequences) + // Calculate the start position of each batch. // example: sequences = {s0, s1, s2} // s0: 0 0 0 0, s1: 1 1 1 1 1, s2: 2 2 2 // num_batch = 5, @@ -95,19 +94,25 @@ class LoDTensor2BatchFunctor { // 6, 2, 11, // 7, 3, // 8} - // The batch number represents batch size after rearranging the + // seq_order = {1, 0, 2}, the sort order. + // where 1 is the second sequence, + // 0 is the first sequence, + // 2 is the third sequence. + // The num_batch represents batch size after rearranging the // input LodTensor. It is also the maximum length of input sequence. paddle::framework::LoD batch_lods; batch_lods.emplace_back(std::vector{0}); batch_lods.emplace_back(std::vector{0}); + batch_lods.emplace_back(std::vector{0}); // batch_lods[0] is the start positions for batch LoDTensor int num_batch = seq_info[0].length; batch_lods[0].resize(static_cast(num_batch + 1)); // batch_lods[1] is the raw index in the input LoDTensor - auto dims = lod_tensor.dims(); - batch_lods[1].resize(static_cast(dims[0])); + batch_lods[1].resize(static_cast(lod_tensor.dims()[0])); + // batch_lods[2] is the sort order for the input LoDTensor. + batch_lods[2].resize(seq_info.size()); size_t* batch_starts = batch_lods[0].data(); size_t* seq2batch_idx = batch_lods[1].data(); @@ -127,6 +132,10 @@ class LoDTensor2BatchFunctor { } batch_starts[n + 1] = static_cast(batch_id); } + size_t* seq_order = batch_lods[2].data(); + for (size_t i = 0; i < seq_info.size(); ++i) { + seq_order[i] = seq_info[i].seq_idx; + } batch.set_lod(batch_lods); CopyMatrixRowsFunctor to_batch; @@ -141,8 +150,7 @@ class Batch2LoDTensorFunctor { const framework::LoDTensor& batch, framework::LoDTensor& lod_tensor) const { auto in_lod = batch.lod(); - PADDLE_ENFORCE_EQ(in_lod.size(), 2UL, - "The LoD size of input `batch` should be 2."); + PADDLE_ENFORCE_GT(in_lod.size(), 2UL); PADDLE_ENFORCE_EQ(in_lod[1].size(), static_cast(lod_tensor.dims()[0])); CopyMatrixRowsFunctor to_seq; diff --git a/paddle/operators/math/sequence_pooling.cc b/paddle/operators/math/sequence_pooling.cc new file mode 100644 index 0000000000000000000000000000000000000000..5913c99fdb01100d0de44ab317124550fa626528 --- /dev/null +++ b/paddle/operators/math/sequence_pooling.cc @@ -0,0 +1,103 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/sequence_pooling.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { +namespace math { + +template +class MaxSeqPoolFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::LoDTensor& input, framework::Tensor* output, + framework::Tensor* index) { + auto in_dims = input.dims(); + auto out_dims = output->dims(); + auto idx_dims = index->dims(); + PADDLE_ENFORCE_GT(in_dims.size(), 1); + PADDLE_ENFORCE_GT(out_dims.size(), 1); + for (int64_t i = 1; i < in_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(in_dims[i], out_dims[i]); + } + PADDLE_ENFORCE_EQ(idx_dims, out_dims); + + auto starts = input.lod()[0]; + const T* in_data = input.data(); + T* out_data = output->data(); + int* max_index = index->data(); + + int64_t num_seq = out_dims[0]; + int64_t dim = output->numel() / num_seq; + for (int64_t i = 0; i < num_seq; ++i) { + for (int64_t k = 0; k < dim; ++k) { + out_data[i * dim + k] = in_data[starts[i] * dim + k]; + max_index[i * dim + k] = starts[i]; + } + for (size_t j = starts[i] + 1; j < starts[i + 1]; ++j) { + for (int64_t k = 0; k < dim; ++k) { + if (in_data[j * dim + k] > out_data[i * dim + k]) { + out_data[i * dim + k] = in_data[j * dim + k]; + max_index[i * dim + k] = j; + } + } + } + } + } +}; + +template +class MaxSeqPoolGradFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& out_grad, + const framework::Tensor& index, + framework::LoDTensor* in_grad) { + auto og_dims = out_grad.dims(); + auto ig_dims = in_grad->dims(); + auto idx_dims = index.dims(); + PADDLE_ENFORCE_GT(og_dims.size(), 1); + PADDLE_ENFORCE_GT(ig_dims.size(), 1); + for (int64_t i = 1; i < og_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(og_dims[i], ig_dims[i]); + } + PADDLE_ENFORCE_EQ(idx_dims, og_dims); + + const T* og_data = out_grad.data(); + const int* max_index = index.data(); + T* ig_data = in_grad->data(); + + SetConstant set_zero; + set_zero(context, in_grad, static_cast(0.0)); + int64_t num_seq = og_dims[0]; + int64_t dim = out_grad.numel() / num_seq; + for (int64_t i = 0; i < num_seq; ++i) { + for (int64_t j = 0; j < dim; ++j) { + int step_id = max_index[i * dim + j]; + ig_data[step_id * dim + j] = og_data[i * dim + j]; + } + } + } +}; + +template class MaxSeqPoolFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolGradFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/sequence_pooling.cu b/paddle/operators/math/sequence_pooling.cu new file mode 100644 index 0000000000000000000000000000000000000000..5ed951402fecba66a8960f4d024bf3785dac51c7 --- /dev/null +++ b/paddle/operators/math/sequence_pooling.cu @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/sequence_pooling.h" + +namespace paddle { +namespace operators { +namespace math { + +#define FLT_MAX __FLT_MAX__ + +template +__global__ void KeMaxSequencePool(const T* input, const size_t* starts, + T* output, int* index, int64_t num_seq, + int64_t dim) { + int dim_idx = threadIdx.x; + int seq_id = blockIdx.x; + if (seq_id >= num_seq) return; + size_t start = starts[seq_id]; + size_t end = starts[seq_id + 1]; + + for (int64_t i = dim_idx; i < dim; i += blockDim.x) { + T max_val = static_cast(-FLT_MAX); + int max_id = -1; + for (size_t step_id = start; step_id < end; step_id++) { + if (max_val < input[step_id * dim + i]) { + max_val = input[step_id * dim + i]; + max_id = step_id; + } + } + output[seq_id * dim + i] = max_val; + index[seq_id * dim + i] = max_id; + } +} + +template +class MaxSeqPoolFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::LoDTensor& input, framework::Tensor* output, + framework::Tensor* index) { + auto in_dims = input.dims(); + auto out_dims = output->dims(); + auto idx_dims = index->dims(); + PADDLE_ENFORCE_GT(in_dims.size(), static_cast(1)); + PADDLE_ENFORCE_GT(out_dims.size(), 1); + for (int64_t i = 1; i < in_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(in_dims[i], out_dims[i]); + } + PADDLE_ENFORCE_EQ(idx_dims, out_dims); + + auto starts = input.lod()[0]; + const T* in_data = input.data(); + T* out_data = output->data(); + int* max_index = index->data(); + + int64_t num_seq = out_dims[0]; + int64_t dim = output->numel() / num_seq; + + dim3 threads(256, 1); + dim3 grid(num_seq, 1); + auto stream = + reinterpret_cast(context).stream(); + KeMaxSequencePool<<>>( + in_data, starts.data(), out_data, max_index, num_seq, dim); + } +}; + +template +__global__ void KeMaxSequencePoolGrad(const T* out_grad, const int* max_index, + T* in_grad, int64_t num_seq, + int64_t dim) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + int col_idx = idx % dim; + if (idx < num_seq * dim) { + int step_id = max_index[idx]; + in_grad[step_id * dim + col_idx] = out_grad[idx]; + } +} + +template +class MaxSeqPoolGradFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& out_grad, + const framework::Tensor& index, + framework::LoDTensor* in_grad) { + auto og_dims = out_grad.dims(); + auto idx_dims = index.dims(); + auto ig_dims = in_grad->dims(); + PADDLE_ENFORCE_GT(og_dims.size(), static_cast(1)); + PADDLE_ENFORCE_GT(ig_dims.size(), static_cast(1)); + for (int64_t i = 1; i < og_dims.size(); ++i) { + PADDLE_ENFORCE_EQ(og_dims[i], ig_dims[i]); + } + PADDLE_ENFORCE_EQ(idx_dims, og_dims); + + const T* og_data = out_grad.data(); + const int* max_index = index.data(); + T* ig_data = in_grad->data(); + + SetConstant set_zero; + set_zero(context, in_grad, static_cast(0.0)); + int64_t num_seq = og_dims[0]; + int64_t dim = out_grad.numel() / num_seq; + + unsigned int blocks = (num_seq * dim + 128 - 1) / 128; + dim3 threads(128, 1); + dim3 grid(blocks, 1); + auto stream = + reinterpret_cast(context).stream(); + KeMaxSequencePoolGrad<<>>( + og_data, max_index, ig_data, num_seq, dim); + } +}; + +template class MaxSeqPoolFunctor; +template class MaxSeqPoolFunctor; +template class MaxSeqPoolGradFunctor; +template class MaxSeqPoolGradFunctor; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/fill_constant_op.h b/paddle/operators/math/sequence_pooling.h similarity index 52% rename from paddle/operators/fill_constant_op.h rename to paddle/operators/math/sequence_pooling.h index 3668f42f1c29541e29463ff3969064e80703fa04..35dfe26de1a87a064410401244914d4e2a94176e 100644 --- a/paddle/operators/fill_constant_op.h +++ b/paddle/operators/math/sequence_pooling.h @@ -13,25 +13,33 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" +#include "paddle/framework/lod_tensor.h" +#include "paddle/framework/tensor.h" +#include "paddle/platform/device_context.h" namespace paddle { namespace operators { +namespace math { + +#define FLT_MAX __FLT_MAX__ template -class FillConstantOpKernel : public framework::OpKernel { +class MaxSeqPoolFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::LoDTensor& input, framework::Tensor* output, + framework::Tensor* index); +}; + +template +class MaxSeqPoolGradFunctor { public: - void Compute(const framework::ExecutionContext& ctx) const override { - auto* out = ctx.Output("Out"); - out->mutable_data(ctx.GetPlace()); - auto value = ctx.Attr("value"); - - auto out_eigen = framework::EigenVector::Flatten(*out); - auto place = ctx.GetEigenDevice(); - out_eigen.device(place) = out_eigen.constant(static_cast(value)); - } + void operator()(const platform::DeviceContext& context, + const framework::Tensor& out_grad, + const framework::Tensor& index, + framework::LoDTensor* in_grad); }; +} // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/matmul_op.cc b/paddle/operators/matmul_op.cc index 5ecbee3b413617e3a5523d9a32e72bc08bd316c5..5a1a6154203d40186f1e41491194b19612931b1f 100644 --- a/paddle/operators/matmul_op.cc +++ b/paddle/operators/matmul_op.cc @@ -144,7 +144,10 @@ class MatMulOpMaker : public framework::OpProtoAndCheckerMaker { )DOC") .SetDefault(false); AddComment(R"DOC( -The MatMul operator is used to perform (batched) matrix multiplication +MatMul Operator. + + +This operator is used to perform (batched) matrix multiplication over the last two dimensions of the input tensors `X` and `Y`. If a transpose flag is specified, the last two dimensions of the @@ -166,7 +169,8 @@ The differences are: - We add `transpose_X` and `transpose_Y` flags. Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/matmul_op.h b/paddle/operators/matmul_op.h index 5ce30740c90b5cd0bd4f8ab183cf985ed5d827c1..4f565946d596b5e5fbf90f16c0c13c780c36886c 100644 --- a/paddle/operators/matmul_op.h +++ b/paddle/operators/matmul_op.h @@ -74,11 +74,10 @@ Tensor CombineBatchAndN(const framework::ExecutionContext& context, Tensor output; auto in_dims = input.dims(); if (in_dims.size() == 3) { - output.Resize(in_dims); + output.Resize({in_dims[1], in_dims[0], in_dims[2]}); output.mutable_data(context.GetPlace()); EigenTranspose(context, input, output, {1, 0, 2}); - std::vector out_dims = {in_dims[1], in_dims[0] * in_dims[2]}; - output.Resize(make_ddim(out_dims)); + output.Resize({in_dims[1], in_dims[0] * in_dims[2]}); } else { output.ShareDataWith(input); } diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 7caa1c9d0cf4dba33a206c85bcbed1fb1cb4e010..dcc5b4286f4ac833268a779a9a7edd2ed119ffff 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -36,7 +36,11 @@ class MeanOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of mean op"); AddOutput("Out", "The output of mean op"); - AddComment(R"DOC( Mean Operator + AddComment(R"DOC( +Mean Operator. + +Out is a scalar which is the mean of all elements in X. + )DOC"); } }; @@ -47,6 +51,7 @@ class MeanGradOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + ctx->ShareLoD("X", framework::GradVarName("X")); } }; diff --git a/paddle/operators/merge_lod_tensor_op.cc b/paddle/operators/merge_lod_tensor_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..80460c476921b63ec5228a9780880c7db3c85217 --- /dev/null +++ b/paddle/operators/merge_lod_tensor_op.cc @@ -0,0 +1,182 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memcpy.h" + +namespace paddle { +namespace operators { + +using LoD = framework::LoD; + +class MergeLoDTensorOp : public framework::OperatorBase { + public: + MergeLoDTensorOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &mask = scope.FindVar(Input("Mask"))->Get(); + auto &in_true = scope.FindVar(Input("InTrue"))->Get(); + auto &in_false = + scope.FindVar(Input("InFalse"))->Get(); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + auto level = static_cast(Attr("level")); + + auto &mask_dim = mask.dims(); + + std::unique_ptr cpu_mask{new framework::LoDTensor()}; + if (platform::is_cpu_place(mask.place())) { + cpu_mask->ShareDataWith(mask); + } else if (platform::is_gpu_place(mask.place())) { +#ifdef PADDLE_WITH_CUDA + cpu_mask->CopyFrom(mask, platform::CPUPlace(), dev_ctx); +#else + PADDLE_THROW("Not supported GPU, Please compile WITH_GPU option"); +#endif + } + auto *mask_data = cpu_mask->data(); + + int rank = in_true.dims().size(); + platform::Place place = in_true.place(); + std::type_index data_type = in_true.type(); + framework::DDim in_true_dims = + framework::slice_ddim(in_true.dims(), 1, rank); + + int64_t batch_size = in_true.dims()[0] + in_false.dims()[0]; + + auto in_true_dim_vec = framework::vectorize(in_true_dims); + in_true_dim_vec.insert(in_true_dim_vec.begin(), batch_size); + + framework::DDim out_dims = framework::make_ddim(in_true_dim_vec); + out->Resize(out_dims); + out->mutable_data(place, data_type); + + auto *out_lod = out->mutable_lod(); + out_lod->clear(); + size_t out_offset = 0; + + // Build LoDTensor `out` + + size_t in_true_idx = 0; + size_t in_false_idx = 0; + for (size_t i = 0; i < static_cast(mask_dim[0]); i++) { + const framework::LoDTensor *input = nullptr; + size_t *in_idx = nullptr; + if (static_cast(mask_data[i]) == 0) { + input = &in_false; + in_idx = &in_false_idx; + } else { + input = &in_true; + in_idx = &in_true_idx; + } + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + input->lod(), *in_idx, (*in_idx) + 1, 0); + auto &lod_length = lod_and_offset.first; + + framework::AppendLoD(out_lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + + PADDLE_ENFORCE_GE(end_offset, start_offset); + size_t len = end_offset - start_offset; + if (len == 0) { + continue; + } + out->Slice(out_offset, out_offset + len) + .CopyFrom(input->Slice(start_offset, end_offset), place, dev_ctx); + out_offset += len; + (*in_idx) += 1; + } + + for (size_t i = 0; i < level; i++) { + out_lod->insert(out_lod->begin(), x.lod()[i]); + } + } +}; + +class MergeLoDTensorOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + MergeLoDTensorOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input LoDTensor, contains complete lod information to " + "construct the output"); + AddInput("Mask", "A bool column vector which mask the input"); + AddInput("InTrue", "The True branch to be merged"); + AddInput("InFalse", "The False branch to be merged"); + AddOutput("Out", "The merged output LoDTensor"); + AddAttr("level", "(int) the specific lod level to rank.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment( + R"DOC( + Merge True and False branches of LoDTensor into a single Output, + with a mask at certain lod level. X is used to obtain complete + lod information. Please refer to SplitLoDTensorOp.)DOC"); + } +}; + +class MergeLoDTensorInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "MergeLoDTensorOp must has input X."); + PADDLE_ENFORCE(context->HasInput("Mask"), + "MergeLoDTensorOp must has input Mask."); + PADDLE_ENFORCE(context->HasInput("InTrue"), + "MergeLoDTensorOp must has input InTrue."); + PADDLE_ENFORCE(context->HasInput("InFalse"), + "MergeLoDTensorOp must has input InFalse."); + PADDLE_ENFORCE(context->HasOutput("Out"), + "MergeLoDTensorOp must has output Out"); + + auto mask_dim = context->GetInputDim("Mask"); + PADDLE_ENFORCE_EQ(mask_dim.size(), 2); + PADDLE_ENFORCE_EQ(mask_dim[1], 1); + + context->SetOutputDim("Out", context->GetInputDim("InTrue")); + } +}; + +class MergeLoDTensorGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("split_lod_tensor"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetInput("Mask", Input("Mask")); + grad_op->SetOutput("OutTrue", InputGrad("InTrue")); + grad_op->SetOutput("OutFalse", InputGrad("InFalse")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(merge_lod_tensor, ops::MergeLoDTensorOp, + ops::MergeLoDTensorOpProtoMaker, + ops::MergeLoDTensorInferShape, ops::MergeLoDTensorGradMaker); diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index f7943e99acc5975d077f2319b6f678cfc693c1f3..4684c20208501a3239fd57b35428946bb52af4a0 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -52,14 +52,16 @@ class MinusOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Y", "The right tensor of minus operator."); AddOutput("Out", "The output tensor of minus operator."); - AddComment(R"DOC(Minus Operator + AddComment(R"DOC( +Minus Operator. Equation: - Out = X - Y + $Out = X - Y$ Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index 7b9e9528952d552a69ffe6a628672901c5c1a7fd..28528848af1f467bf38be53f9d05fee6ca3f93cc 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -43,27 +43,35 @@ class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "The input tensor of modified huber loss op." + "The input tensor of modified huber loss op. " "X is 2-D tensor with shape [batch_size, 1]."); AddInput("Y", - "The target labels of modified huber loss op." - "The shape of Y is same as X. Values of Y must be 0 or 1."); + "The target labels of modified huber loss op. " + "The shape of Y is the same as X. Values of Y must be 0 or 1."); AddOutput("IntermediateVal", "Variable to save intermediate result which will be reused in " "backward processing.") .AsIntermediate(); AddOutput("Out", "Classification loss for X."); AddComment(R"DOC( -Modified huber loss is used in binary classification problem. The shape of -input X and target Y are both [N, 1] and so is the shape of output loss. -Since target Y is not differentiable, cacluating gradient for Y is illegal. -The formulation of modified huber loss is: - -L(y, f(x)) = max(0, 1 - yf(x))^2 for yf(x) >= -1, - -4yf(x) otherwise. - -Make sure the values of target label Y are in {0, 1} here. The operator will +Modified Huber Loss Operator. + +This operator is used in binary classification problem. The shape of +input X and target Y are both [N, 1] and so is the shape of the output loss. +Since target Y is not differentiable, calculating gradient for Y is illegal. +The formula of modified huber loss is: + +$$ +L(y, f(x)) = +\begin{cases} +(\max(0, 1 - yf(x)))^2, \text{if} \ yf(x) >= -1 \\ + -4yf(x), \quad \text{otherwise} +\end{cases} +$$ + +Make sure the values of target label Y are in {0, 1} here. This operator will scale values of Y to {-1, +1} when computing losses and gradients. + )DOC"); } }; diff --git a/paddle/operators/momentum_op.cc b/paddle/operators/momentum_op.cc index 2d4d6f13720f0e6888edbddcb3243116506227ba..19954006195c1e9fd34328b52ed2a9eade526235 100644 --- a/paddle/operators/momentum_op.cc +++ b/paddle/operators/momentum_op.cc @@ -75,17 +75,23 @@ class MomentumOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("VelocityOut", "(Tensor) Output updated velocity"); AddAttr("mu", "(float) Momentum coefficient"); - AddAttr("useNesterov", "(bool) Use Nesterov Momentum") + AddAttr("use_nesterov", + "(bool, default false) " + "Use Nesterov Momentum") .SetDefault(false); AddComment(R"DOC( - -Momentum Algorithm with a flag for Nestrov Moemntum (momentum). - -velocity = mu * velocity + gradient -if (use_nesterov): - param = param - gradient * learning_rate + mu * velocity * learning_rate -else: - param = param - learning_rate * velocity +Momentum Optimizer. + +This optimizer has a flag for Nestrov Momentum. +The update equations are as follows: + +$$ +velocity = mu * velocity + gradient \\ +if (use\_nesterov): \\ + param = param - gradient * learning\_rate + mu * velocity * learning\_rate \\ +else: \\ + param = param - learning\_rate * velocity. \\ +$$ )DOC"); } diff --git a/paddle/operators/momentum_op.h b/paddle/operators/momentum_op.h index e6d6d1da3df9f7e43a93fcc2e12658a01a491f81..8f7f5eb5c21c0342f57a47b85d28f4454f4566c2 100644 --- a/paddle/operators/momentum_op.h +++ b/paddle/operators/momentum_op.h @@ -34,7 +34,7 @@ class MomentumOpKernel : public framework::OpKernel { velocity_out->mutable_data(ctx.GetPlace()); float mu = ctx.Attr("mu"); - bool use_nesterov = ctx.Attr("useNesterov"); + bool use_nesterov = ctx.Attr("use_nesterov"); auto p_out = framework::EigenVector::Flatten(*param_out); auto v_out = framework::EigenVector::Flatten(*velocity_out); diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 245d3b47d3a6331a3cf20dbdbd972639d68cd496..3c39ae10dc50084cff284c307167c33c9208a3ce 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -29,9 +29,14 @@ class MulOpShapeInference : public framework::InferShapeBase { auto x_dims = ctx->GetInputDim("X"); auto y_dims = ctx->GetInputDim("Y"); + int x_num_col_dims = ctx->Attrs().Get("x_num_col_dims"); int y_num_col_dims = ctx->Attrs().Get("y_num_col_dims"); + VLOG(3) << "mul operator x.shape=" << x_dims << " y.shape=" << y_dims + << " x_num_col_dims=" << x_num_col_dims + << " y_num_col_dims=" << y_num_col_dims; + PADDLE_ENFORCE_GT( x_dims.size(), x_num_col_dims, "The input tensor X's rank of MulOp should be larger than " @@ -73,6 +78,7 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "The output of mul op"); AddAttr( "x_num_col_dims", + "(int, default 1) " R"DOC(mul_op can take tensors with more than two dimensions as input `X`, in that case, tensors will be reshaped to a matrix. The matrix's first dimension(column length) will be the product of tensor's last @@ -83,20 +89,24 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { .EqualGreaterThan(1); AddAttr( "y_num_col_dims", + "(int, default 1) " R"DOC(mul_op can take tensors with more than two dimensions as input `Y`, in that case, tensors will be reshaped to a matrix. Just like input `X`. )DOC") .SetDefault(1) .EqualGreaterThan(1); AddComment(R"DOC( -Mul operator is used to perform matrix multiplication for input X and Y. +Mul Operator. + +This operator is used to perform matrix multiplication for input X and Y. The equation is: - Out = X * Y + $$Out = X * Y$$ Both the input `X` and `Y` can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with input `X`. +or not. But the output only shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/mul_op.cu b/paddle/operators/mul_op.cu index a81444dbe63edeecedc5d822c65ff56c42b5db90..66dc3d6d106a18640adad413d4e967fa101abcfc 100644 --- a/paddle/operators/mul_op.cu +++ b/paddle/operators/mul_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/operators/mul_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/mul_op.h b/paddle/operators/mul_op.h index bd1bdb4f81b88256822d663fe42ad314338c91ff..0eb9df41e9415845f88af283de63856158b447f9 100644 --- a/paddle/operators/mul_op.h +++ b/paddle/operators/mul_op.h @@ -16,16 +16,12 @@ #include "paddle/operators/math/math_function.h" -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; template class MulKernel : public framework::OpKernel { diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc index 4d86769026e4b3e3040bdcb3bc6dc2edea58b4b0..f8527dfab3f3c42f430c433a11351f12b8dfae8b 100644 --- a/paddle/operators/multiplex_op.cc +++ b/paddle/operators/multiplex_op.cc @@ -51,9 +51,11 @@ class MultiplexOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.MultiInput("X")[0]->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.MultiInput("X")[0]->type()), + ctx.device_context()); } }; @@ -66,7 +68,8 @@ class MultiplexOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("X", "The candidate tensors of multiplex operator.") .AsDuplicable(); AddOutput("Out", "The output tensor of multiplex operator."); - AddComment(R"DOC(Multiplex operator + AddComment(R"DOC( +Multiplex Operator. Multiplex multiple tensors according to the index provided by the index tensor. @@ -77,10 +80,11 @@ the (Ids[i])-th tensor. For i-th row of the output tensor: -y[i] = x_{k}[i] +$$y[i] = x_{k}[i]$$ -where y is the output tensor. `x_{k}` is the k-th input tensor +where `y` is the output tensor, `x_{k}` is the k-th input tensor, and `k = Ids[i]`. + )DOC"); } }; @@ -105,9 +109,11 @@ class MultiplexGradOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.MultiInput("X")[0]->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.MultiInput("X")[0]->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/multiplex_op.cu b/paddle/operators/multiplex_op.cu index 143a14fef5783f8ed085d4c4ce2afb3b190d0600..49ed8a8879527fd32dd8b001ea256e46a0353487 100644 --- a/paddle/operators/multiplex_op.cu +++ b/paddle/operators/multiplex_op.cu @@ -35,9 +35,7 @@ class MultiplexGPUKernel : public framework::OpKernel { Tensor index_t_cpu; index_t_cpu.CopyFrom(*ids, platform::CPUPlace(), ctx.device_context()); auto* index = index_t_cpu.data(); - auto stream = reinterpret_cast( - ctx.device_context()) - .stream(); + auto stream = ctx.cuda_device_context().stream(); Place place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { int32_t k = index[i]; @@ -73,9 +71,7 @@ class MultiplexGradGPUKernel : public framework::OpKernel { index_t_cpu.CopyFrom(*ids, platform::CPUPlace(), ctx.device_context()); auto* index = index_t_cpu.data(); - auto stream = reinterpret_cast( - ctx.device_context()) - .stream(); + auto stream = ctx.cuda_device_context().stream(); Place place = boost::get(ctx.GetPlace()); for (auto i = 0; i < rows; i++) { size_t k = static_cast(index[i]); diff --git a/paddle/operators/name_convention.md b/paddle/operators/name_convention.md index 5a216907950100070ba57176c382eb659effb293..b5cb176e003b4584321142ac9f1c3380b7010936 100644 --- a/paddle/operators/name_convention.md +++ b/paddle/operators/name_convention.md @@ -4,10 +4,10 @@ To make the operator document itself more clear, we recommend operator names obe ### OpProtoMaker names -When defining an operator in Paddle, a corresponding [OpProtoMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L170) (TODO: OpProtoMaker Doc)need to be defined. All the Input/Output and Attributes will write into the [OpProto](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L61) , and will be used in client language to create operator. +When defining an operator in Paddle, a corresponding [OpProtoMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L170) (TODO: OpProtoMaker Doc)need to be defined. All the Input/Output and Attributes will write into the [OpProto](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L61) , and will be used in client language to create operator. - Input/Output. - - Input/Output names follow the **CamelCase**. e.g. `X`, `Y`, `Matrix`, `LastAxisInMatrix`. Input/Output much more like Variables, we prefer to meaningful English words. + - Input/Output names follow the **CamelCase**. e.g. `X`, `Y`, `Matrix`, `LastAxisInMatrix`. Input/Output much more like Variables, we prefer to meaningful English words. - If an operator's Input/Output are tensors in math, not match to any meaningful words, input name should starts from `X`. e.g. `X`, `Y`, and output name should starts from `Out`. e.g. `Out`. This rule intends making operators which have few inputs/outputs unified. - Attribute. @@ -15,7 +15,7 @@ When defining an operator in Paddle, a corresponding [OpProtoMaker](https://gith - Comments. - Input/Output/Attr comment follow the format of **(type,default value) usage**, corresponding to which type it can be and how it will be used in the operator. e.g. Attribute in Accumulator`"gamma" `,`(float, default 1.0) Accumulation multiplier`. - - Operator comment format of` R"DOC(your comment here)DOC"`. You should explain the input/output of the operator first. If there is math calculation in this operator, you should write the equation in the comment. e.g. `Out = X + Y`. + - Operator comment format of` R"DOC(your comment here)DOC"`. You should explain the input/output of the operator first. If there is math calculation in this operator, you should write the equation in the comment. e.g. `Out = X + Y`. - Order. - Follow the order of Input/Output, then Attribute, then Comments. See the example in best practice. @@ -24,7 +24,7 @@ When defining an operator in Paddle, a corresponding [OpProtoMaker](https://gith Here we give some examples to show how these rules will be used. -- The operator has one input, one output. e.g.`relu`, inputs: `X`, outputs: `Out`. +- The operator has one input, one output. e.g.`relu`, inputs: `X`, outputs: `Out`. - The operator has two input, one output. e.g. `rowwise_add`, inputs : `X`, `Y`, outputs : `Out`. @@ -38,23 +38,27 @@ public: AccumulateOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "(Tensor) The input tensor that has to be accumulated to the output tensor. - If the output size is not the same as input size, + AddInput("X", "(Tensor) The input tensor that has to be accumulated to the output tensor. + If the output size is not the same as input size, the output tensor is first reshaped and initialized to zero, and only then, accumulation is done."); AddOutput("Out", "(Tensor) Accumulated output tensor"); AddAttr("gamma", "(float, default 1.0) Accumulation multiplier").SetDefault(1.0f); AddComment(R"DOC( -Accumulate operator accumulates the input tensor to the output tensor. If the +Accumulate Operator. + +This operator accumulates the input tensor to the output tensor. If the output tensor already has the right size, we add to it; otherwise, we first initialize the output tensor to all zeros, and then do accumulation. Any further calls to the operator, given that no one else fiddles with the output in the interim, will do simple accumulations. -Accumulation is done as shown: + +Accumulation is done as follows: Out = 1*X + gamma*Out where X is the input tensor, Out is the output tensor and gamma is the multiplier argument. + )DOC"); } }; diff --git a/paddle/operators/nccl/nccl_gpu_common.h b/paddle/operators/nccl/nccl_gpu_common.h index 5858cd4839d367bb888b2b98cde2225751391162..48e322f99398a7f1d6af9cab653d0cc92d981fe0 100644 --- a/paddle/operators/nccl/nccl_gpu_common.h +++ b/paddle/operators/nccl/nccl_gpu_common.h @@ -35,6 +35,7 @@ constexpr int kInvalidGPUId = -1; struct Communicator { std::vector comms_; std::unordered_map comm_id_map_; + bool inited_; Communicator() {} @@ -42,17 +43,21 @@ struct Communicator { void InitAll(const std::vector& gpus) { comms_.resize(gpus.size()); + inited_ = false; for (size_t i = 0; i < gpus.size(); ++i) { comm_id_map_[gpus[i]] = i; } PADDLE_ENFORCE( dynload::ncclCommInitAll(comms_.data(), gpus.size(), gpus.data())); + inited_ = true; } ~Communicator() { - for (size_t i = 0; i < comms_.size(); ++i) { - // FIXME(dzh) : PADDLE_ENFORCE return void - dynload::ncclCommDestroy(comms_[i]); + if (inited_) { + for (size_t i = 0; i < comms_.size(); ++i) { + // FIXME(dzh) : PADDLE_ENFORCE return void + dynload::ncclCommDestroy(comms_[i]); + } } } diff --git a/paddle/operators/nccl_op.cc b/paddle/operators/nccl_op.cc index d39cb2fcf9cc205edf86f8ab1d5e04b5672e00f6..66fcc09bc877867e66a37adc73230d8dabf4cbed 100644 --- a/paddle/operators/nccl_op.cc +++ b/paddle/operators/nccl_op.cc @@ -48,12 +48,17 @@ class NCCLInitOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddOutput("Communicator", "Create Communicator for communicating between gpus"); - AddAttr>("gpus", "gpu id lists"); - AddAttr("data_type", "output data type") + AddAttr>("gpus", "(vector) GPU id lists"); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") .SetDefault(framework::DataType::FP32); AddComment(R"DOC( - create communicator. - )DOC"); +NCCLInit Operator. + +Create communicator. + +)DOC"); } }; @@ -143,11 +148,15 @@ class NCCLAllReduceOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of AllReduce op"); AddAttr("reduction", + "(string, default 'ncclSum') " "{'ncclMin', 'ncclMax', 'ncclProd', 'ncclSum'}.") .SetDefault("ncclSum"); AddComment(R"DOC( - AllReduce the input tensors. - )DOC"); +NCCLAllReduce Operator. + +AllReduce the input tensors. + +)DOC"); } }; @@ -161,14 +170,20 @@ class NCCLReduceOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of Reduce op"); AddAttr("reduction", + "(string, default 'ncclSum') " "{'ncclMin', 'ncclMax', 'ncclProd', 'ncclSum'}.") .SetDefault("ncclSum"); AddAttr("root", - "root gpu of the parameter. if not " - "set(platform::kInvalidGPUId). hashed by name.") + "(int, default kInvalidGPUId) " + "Root gpu of the parameter. If not, " + "set(platform::kInvalidGPUId). Hashed by name.") .SetDefault(platform::kInvalidGPUId); AddComment(R"DOC( - Reduce the tensors)DOC"); +NCCLReduce Operator. + +Reduce the tensors. + +)DOC"); } }; @@ -182,12 +197,16 @@ class NCCLBcastOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Communicator", "Communicator for communicating between gpus"); AddOutput("Out", "The output of Bcast"); AddAttr("root", - "root gpu of the parameter. if not " - "set(platform::kInvalidGPUId). hashed by name.") + "(int, default kInvalidGPUId) " + "Root gpu of the parameter. If not, " + "set(platform::kInvalidGPUId). Hashed by name.") .SetDefault(platform::kInvalidGPUId); AddComment(R"DOC( - Bcast the tensors. - )DOC"); +NCCLBcast Operator. + +Bcast the tensors. + +)DOC"); } }; diff --git a/paddle/operators/nccl_op.cu b/paddle/operators/nccl_op.cu index 86dee8ee8e1c1a1041d6bc9fa515d669a9c4e466..4f0a2a79edb9f24c7758fc91483d374425b36853 100644 --- a/paddle/operators/nccl_op.cu +++ b/paddle/operators/nccl_op.cu @@ -64,9 +64,7 @@ class NCCLAllReduceKernel : public framework::OpKernel { auto* comm = ctx.Input("Communicator"); - auto stream = reinterpret_cast( - ctx.device_context()) - .stream(); + auto stream = ctx.cuda_device_context().stream(); // device id int gpu_id = boost::get(ctx.GetPlace()).GetDeviceId(); diff --git a/paddle/operators/nccl_op_test.cu b/paddle/operators/nccl_op_test.cu index e5927d56ae7cfbd09e941c993041af46ecd8d70d..56ba57854955c08031214d1f751c17fbb8bb882c 100644 --- a/paddle/operators/nccl_op_test.cu +++ b/paddle/operators/nccl_op_test.cu @@ -26,7 +26,6 @@ #include "paddle/framework/op_registry.h" #include "paddle/framework/program_desc.h" #include "paddle/framework/var_desc.h" -#include "paddle/operators/math/math_function.h" #include "paddle/operators/nccl/nccl_gpu_common.h" #include "paddle/platform/device_context.h" #include "paddle/platform/enforce.h" diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 73a0b8baff530840ddd0d4c65cd4c060ab18e401..adb75df6ef10c59fc6f3db4d36e1ffb1ae0b4b1e 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -54,41 +54,44 @@ class PadOpMaker : public framework::OpProtoAndCheckerMaker { "The input of pad op. " "The input should be a k-D tensor(k > 0 and k < 7)"); AddOutput("Out", - "The output of pad op." + "The output of pad op. " "A tensor with the same shape as X."); + AddAttr>( + "paddings", + "(vector) " + "A list to describe the padding rules for each dimension. " + "For 2-D image tensor, paddings=[0, 1, 2, 3] means " + "padding 0 row to top, 1 row to bottom, 2 columns to left " + "and 3 columns to right. Size of paddings should be equal to " + "2 * dimension size of the input tensor."); + AddAttr("pad_value", + "(float, default 0.0) " + "The value to fill the padded areas.") + .SetDefault(0.0f); AddComment(R"DOC( -Pad input into output, as specified by paddings and pad_value. The input should be a k-D tensor(k > 0 and k < 7). As an example: +Pad Operator. + +Pad input into output, as specified by paddings and pad_value. +The input should be a k-D tensor(k > 0 and k < 7). As an example: Given: X = [[1, 2], - [3, 4]] - -and + [3, 4]], -paddings = [0, 1, 1, 2] +paddings = [0, 1, 1, 2], and -pad_value = 0 +pad_value = 0, -then we get +we have: Out = [[0, 1, 2, 0, 0] [0, 3, 4, 0, 0] [0, 0, 0, 0, 0]] + )DOC"); - AddAttr>( - "paddings", - "A list to describes padding rules for each dimension." - " For 2-D image tensor, paddings=[0, 1, 2, 3] means" - " padding 0 row to top, 1 row to bottom, 2 columns to left" - " and 3 columns to right.Size of paddings should be equal to" - " 2 * dimension size of input tensor."); - AddAttr("pad_value", - "(float) default to 0; " - "The value to fill padded areas.") - .SetDefault(0.0f); } }; diff --git a/paddle/operators/pool_cudnn_op.cu b/paddle/operators/pool_cudnn_op.cu index 8d0741dccc1fdae069af55da49f44378e2c4ddf8..8711567b95fea355396173b5312d26d31f9ffb12 100644 --- a/paddle/operators/pool_cudnn_op.cu +++ b/paddle/operators/pool_cudnn_op.cu @@ -37,11 +37,11 @@ class PoolCudnnOpKernel : public framework::OpKernel { const T *input_data = input->data(); T *output_data = output->mutable_data(ctx.GetPlace()); - std::string pooling_type = ctx.Attr("poolingType"); + std::string pooling_type = ctx.Attr("pooling_type"); std::vector ksize = ctx.Attr>("ksize"); std::vector strides = ctx.Attr>("strides"); std::vector paddings = ctx.Attr>("paddings"); - if (ctx.Attr("globalPooling")) { + if (ctx.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(input->dims()[i + 2]); @@ -92,12 +92,12 @@ class PoolCudnnGradOpKernel : public framework::OpKernel { ctx.Input(framework::GradVarName("Out")); Tensor *input_grad = ctx.Output(framework::GradVarName("X")); - std::string pooling_type = ctx.Attr("poolingType"); + std::string pooling_type = ctx.Attr("pooling_type"); std::vector ksize = ctx.Attr>("ksize"); std::vector strides = ctx.Attr>("strides"); std::vector paddings = ctx.Attr>("paddings"); - if (ctx.Attr("globalPooling")) { + if (ctx.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(input->dims()[i + 2]); diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 4d75c11bc8130343e95f75e687529303179caa93..f3963b1995ef8767786f0bf230b134afc69aa99d 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -29,7 +29,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { auto in_x_dims = ctx->GetInputDim("X"); - std::string pooling_type = ctx->Attrs().Get("poolingType"); + std::string pooling_type = ctx->Attrs().Get("pooling_type"); std::vector ksize = ctx->Attrs().Get>("ksize"); std::vector strides = ctx->Attrs().Get>("strides"); std::vector paddings = ctx->Attrs().Get>("paddings"); @@ -37,7 +37,7 @@ void PoolOp::InferShape(framework::InferShapeContext *ctx) const { PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5, "Pooling intput should be 4-D or 5-D tensor."); - if (ctx->Attrs().Get("globalPooling")) { + if (ctx->Attrs().Get("global_pooling")) { ksize.resize(static_cast(in_x_dims.size()) - 2); for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; @@ -73,125 +73,139 @@ Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto, AddInput( "X", "(Tensor) The input tensor of pooling operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of feature."); + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of channels, H is the height of the feature, " + "and W is the width of the feature."); AddOutput("Out", - "(Tensor) The output tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is " - "the number of channels, H and W is the height and " - "width of feature."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCHW, " + "where N is batch size, C is the number of channels, " + "H is the height of the feature, " + "and W is the width of the feature."); - AddAttr("poolingType", + AddAttr("pooling_type", "(string), pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); AddAttr>("ksize", - "(vector ), the pooling window size(height, width) " - "of pooling operator." - "If globalPooling = true, ksize and paddings will " + "(vector) The pooling window " + "size(height, width) of the pooling operator. " + "If global_pooling = true, ksize and paddings will " "be ignored."); // TODO(Chengduo): Add checker. // (Currently, // TypedAttrChecker don't support vector type.) - AddAttr("globalPooling", - "(bool default: false), whether to use the global pooling." - "If globalPooling = true, ksize and paddings will be ignored.") + AddAttr("global_pooling", + "(bool, default false) Whether to use the global pooling. " + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); - AddAttr>( - "strides", - "(vector, default:{1, 1}), strides(height, width) of pooling operator.") + AddAttr>("strides", + "(vector, default {1, 1}), strides(height, " + "width) of pooling operator.") .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0}), paddings(height, width) of pooling operator." - "If globalPooling = true, paddings and ksize will be ignored.") + "(vector, defalut {0,0}), paddings(height, width) of pooling " + "operator." + "If global_pooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +Pool2d Operator. + The pooling2d operation calculates the output based on -the input, poolingType and ksize, strides, paddings parameters. -Input(X) and output(Out) are in NCHW format. Where N is batch size, C is the -number of channels, H and W is the height and width of feature. +the input, pooling_type and ksize, strides, paddings parameters. +Input(X) and output(Out) are in NCHW format, where N is batch size, C is the +number of channels, H is the height of the feature, and W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively. The input(X) size and output(Out) size may be different. Example: Input: - X shape: (N, C, H_in, W_in) + X shape: $(N, C, H_{in}, W_{in})$ Output: - Out shape: (N, C, H_out, W_out) - where - H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + Out shape: $(N, C, H_{out}, W_{out})$ + where + $$ + H_{out} = (H_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + W_{out} = (W_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + $$ + )DOC"); } Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor) The input tensor of pooling operator. " - "The format of input tensor is NCDHW. Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and width of " - "feature."); + AddInput("X", + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCDHW, where N is batch size, C is " + "the number of channels, and D, H and W is the depth, height and " + "width of " + "the feature, respectively."); AddOutput("Out", "(Tensor) The output tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and " - "width of feature."); + "The format of output tensor is also NCDHW, " + "where N is batch size, C is " + "the number of channels, and D, H and W is the depth, height and " + "width of the feature, respectively."); - AddAttr("poolingType", - "(string), pooling type, can be \"max\" for max-pooling " + AddAttr("pooling_type", + "(string) Pooling type, can be \"max\" for max-pooling " "and \"avg\" for average-pooling.") .InEnum({"max", "avg"}); - AddAttr>("ksize", - "(vector ), the pooling window size(depth, height, " - "width) of pooling " - "operator." - "If globalPooling = true, ksize and paddings wille " - "be ignored."); // TODO(Chengduo): Add checker. - // (Currently, + AddAttr>( + "ksize", + "(vector) The pooling window size(depth, height, " + "width) of pooling operator. " + "If global_pooling = true, ksize and paddings will " + "be ignored."); // TODO(Chengduo): Add checker. + // (Currently, // TypedAttrChecker don't support vector type.) - AddAttr("globalPooling", - "(bool default: false), whether to use the global pooling." - "If globalPooling = true, ksize and paddings wille be ignored.") + AddAttr( + "global_pooling", + "(bool, default false) Whether to use the global pooling. " + "If global_pooling = true, ksize and paddings wille be ignored.") .SetDefault(false); - AddAttr>("strides", - "(vector, default:{1,1,1}), strides(depth, height, " - "width) of pooling operator.") + AddAttr>( + "strides", + "(vector, default {1,1,1}) Strides(depth, height, " + "width) of the pooling operator.") .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0,0}), paddings(depth, height, " - "width) of pooling operator." - "If globalPooling = true, ksize and paddings wille be ignored.") + "(vector, defalut {0,0,0}), paddings(depth, height, " + "width) of pooling operator. " + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +Pool3d Operator. + The pooling3d operation calculates the output based on -the input, poolingType and ksize, strides, paddings parameters. -Input(X) and output(Out) are in NCDHW format. Where N is batch -size, C is the number of channels, D, H and W is the depth, height and -width of feature. Parameters(ksize, strides, paddings) are three elements. -These three elements represent depth, height and width, respectively. -The input(X) size and output(Out) size may be different. +the input, pooling_type, ksize, strides, and paddings parameters. +Input(X) and output(Out) are in NCDHW format, where N is batch +size, C is the number of channels, and D, H and W are the depth, height and +width of the feature, respectively. Parameters(ksize, strides, paddings) +are three elements. These three elements represent depth, height and +width, respectively. The input(X) size and output(Out) size may be different. Example: Input: - X shape: (N, C, D_in, H_in, W_in) + X shape: $(N, C, D_{in}, H_{in}, W_{in})$ Output: - Out shape: (N, C, D_out, H_out, W_out) + Out shape: $(N, C, D_{out}, H_{out}, W_{out})$ where - D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1; + $$ + D_{out} = (D_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + H_{out} = (H_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 \\ + W_{out} = (W_{in} - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + $$ + )DOC"); } } // namespace operators diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index d9d445f6a6257b0c8a1959c64c9a878539e10cd4..4da1941ab541483e706257667b14aa5a95e0c3cc 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -57,11 +57,11 @@ class PoolKernel : public framework::OpKernel { const Tensor* in_x = context.Input("X"); Tensor* out = context.Output("Out"); - std::string pooling_type = context.Attr("poolingType"); + std::string pooling_type = context.Attr("pooling_type"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); @@ -119,12 +119,12 @@ class PoolGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); Tensor* in_x_grad = context.Output(framework::GradVarName("X")); - std::string pooling_type = context.Attr("poolingType"); + std::string pooling_type = context.Attr("pooling_type"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc index 95e896e7cc33b1aebe78d1af8746a25318048041..1df36e965abab3549aeb88bf682b712033c4d79c 100644 --- a/paddle/operators/pool_with_index_op.cc +++ b/paddle/operators/pool_with_index_op.cc @@ -44,7 +44,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5, "Pooling intput should be 4-D or 5-D tensor."); - if (ctx->Attrs().Get("globalPooling")) { + if (ctx->Attrs().Get("global_pooling")) { ksize.resize(static_cast(in_x_dims.size()) - 2); for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; @@ -89,64 +89,73 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", - "(Tensor), the input tensor of pooling operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of image."); + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCHW, where N is batch size, C is the " + "number of channels, H is the height of the image, " + "and W is the width of the image."); AddOutput("Out", - "(Tensor), the output tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is " - "the number of channels, H and W is the height and " - "width of image."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCHW, " + "where N is batch size, C is " + "the number of channels, H is the height of the image " + "and W is the width of the image."); AddOutput("Mask", - "(Tensor), the Mask tensor of pooling operator." - "The format of output tensor is also NCHW." - "Where N is batch size, C is the number of channels, H and W " - "is the height and width of image." - "The value in it is the index in current feature map"); + "(Tensor) The Mask tensor of pooling operator." + "The format of output tensor is also NCHW, " + "where N is batch size, C is the number of channels, " + "H is the height of the image, " + "and W is the width of the image. " + "It represents the index in the current feature map."); AddAttr>("ksize", - "(vector ), the pooling window size(height, " - "width) of pooling operator." - "If globalPooling = true, ksize and paddings " + "(vector) The pooling window size(height, " + "width) of pooling operator. " + "If global_pooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( - "globalPooling", - "(bool default: false), whether to use the global pooling." - "If globalPooling = true, ksize and paddings will be ignored.") + "global_pooling", + "(bool, default false) Whether to use the global pooling. " + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); - AddAttr>( - "strides", - "(vector, default:{1, 1}), strides(height, width) of pooling operator.") + AddAttr>("strides", + "(vector, default {1, 1}), strides(height, " + "width) of pooling operator.") .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0, 0}), paddings(height, width) of pooling operator." - "If globalPooling = true, paddings and will be ignored.") + "(vector, defalut {0, 0}), paddings(height, width) of pooling " + "operator. " + "If global_pooling = true, paddings and will be ignored.") .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +MaxPool2d Operator. + The maxPooling2d with index operation calculates the output and the mask -based on the input and ksize, strides, paddings parameters. Input(X) and -output(Out, Mask) are in NCHW format. Where N is batch size, C is the -number of channels, H and W is the height and width of feature. +based on the input, ksize, strides, and paddings parameters. Input(X) and +output(Out, Mask) are in NCHW format, where N is batch size, C is the +number of channels, H is the height of the feature, +and W is the width of the feature. Parameters(ksize, strides, paddings) are two elements. These two elements represent height and width, respectively. The input(X) size and output(Out, Mask) size may be different. Example: Input: - X shape: (N, C, H_in, W_in) + X shape: $(N, C, H_{in}, W_{in})$ Output: - Out shape: (N, C, H_out, W_out) - Mask shape: (N, C, H_out, W_out) + Out shape: $(N, C, H_{out}, W_{out})$ + Mask shape: $(N, C, H_{out}, W_{out})$ where - H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + $$ + H_{out} = (H_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + W_{out} = (W_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + $$ + )DOC"); } }; @@ -156,70 +165,76 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker { MaxPool3dWithIndexOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor), the input tensor of pooling operator. " - "The format of input tensor is NCDHW. Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and width of " - "image."); + AddInput("X", + "(Tensor) The input tensor of pooling operator. " + "The format of input tensor is NCDHW, where N is batch size, C is " + "the number of channels, and D, H and W are the depth, height and " + "width of " + "the image, respectively"); AddOutput("Out", - "(Tensor), the output tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is " - "the number of channels, D, H and W is the depth, height and " - "width of image."); + "(Tensor) The output tensor of pooling operator. " + "The format of output tensor is also NCDHW, " + "where N is the batch size, C is the number of channels, " + "and D, H and W are the depth, height and " + "width of the image, respectively."); AddOutput("Mask", - "(Tensor), the Mask tensor of pooling operator." - "The format of output tensor is also NCDHW." - "Where N is batch size, C is the number of channels, D, H and W " - "is the depth, height and width of image." - "The value in it is the index in current feature map"); + "(Tensor) The Mask tensor of pooling operator. " + "The format of output tensor is also NCDHW, " + "where N is the batch size, C is the number of channels, and " + "D, H and W are the depth, height and width " + "of the image, respectively. " + "It represents the index in the current feature map."); AddAttr>("ksize", - "(vector), the pooling window size(depth, " - "height, width) of pooling " - "operator." - "If globalPooling = true, ksize and paddings " + "(vector) The pooling window size(depth, " + "height, width) of pooling operator. " + "If global_pooling = true, ksize and paddings " "will be ignored."); // TODO(Chengduo): Add // checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr( - "globalPooling", - "(bool default: false), whether to use the global pooling." - "If globalPooling = true, ksize and paddings will be ignored.") + "global_pooling", + "(bool, default false) Whether to use the global pooling. " + "If global_pooling = true, ksize and paddings will be ignored.") .SetDefault(false); AddAttr>("strides", - "(vector, default:{1,1,1}), strides(depth, " + "(vector, default {1,1,1}), strides(depth, " "height, width) of pooling operator.") .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddAttr>( "paddings", - "(vector defalut:{0,0,0}), paddings(depth, " - "height, width) of pooling operator." - "If globalPooling = true, paddings and ksize will be ignored.") + "(vector, defalut {0,0,0}), paddings(depth, " + "height, width) of pooling operator. " + "If global_pooling = true, paddings and ksize will be ignored.") .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently, // TypedAttrChecker don't support vector type.) AddComment(R"DOC( +MaxPool3d Operator. + The maxpooling3d with index operation calculates the output and the mask based on the input and ksize, strides, paddings parameters. -Input(X) and output(Out, Mask) are in NCDHW format. Where N is batch -size, C is the number of channels, D, H and W is the depth, height and -width of feature. Parameters(ksize, strides, paddings) are three elements. +Input(X) and output(Out, Mask) are in NCDHW format, where N is batch +size, C is the number of channels, and D, H and W are the depth, height and +width of the feature, respectively. +Parameters(ksize, strides, paddings) are three elements. These three elements represent depth, height and width, respectively. The input(X) size and output(Out, Mask) size may be different. Example: Input: - X shape: (N, C, D_in, H_in, W_in) + X shape: $(N, C, D_{in}, H_{in}, W_{in})$ Output: - Out shape: (N, C, D_out, H_out, W_out) - Mask shape: (N, C, D_out, H_out, W_out) + Out shape: $(N, C, D_{out}, H_{out}, W_{out})$ + Mask shape: $(N, C, D_{out}, H_{out}, W_{out})$ where - D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1; + $$ + D_{out} = (D_{in} - ksize[0] + 2 * paddings[0]) / strides[0] + 1 \\ + H_{out} = (H_{in} - ksize[1] + 2 * paddings[1]) / strides[1] + 1 \\ + W_{out} = (W_{in} - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + $$ + )DOC"); } }; diff --git a/paddle/operators/pool_with_index_op.h b/paddle/operators/pool_with_index_op.h index 48627740435b7d397c5a53491c1f89ba1b603803..ea37de84abeb577461ccd5c1f0eda8bacb4458eb 100644 --- a/paddle/operators/pool_with_index_op.h +++ b/paddle/operators/pool_with_index_op.h @@ -35,7 +35,7 @@ class MaxPoolWithIndexKernel : public framework::OpKernel { std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x->dims()[i + 2]); @@ -72,7 +72,7 @@ class MaxPoolWithIndexGradKernel : public framework::OpKernel { std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (context.Attr("globalPooling")) { + if (context.Attr("global_pooling")) { for (size_t i = 0; i < ksize.size(); ++i) { paddings[i] = 0; ksize[i] = static_cast(in_x_grad->dims()[i + 2]); diff --git a/paddle/operators/positive_negative_pair_op.cc b/paddle/operators/positive_negative_pair_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..4ba40a62ec5f696ad980c2913f7e162879a557e2 --- /dev/null +++ b/paddle/operators/positive_negative_pair_op.cc @@ -0,0 +1,179 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/positive_negative_pair_op.h" + +namespace paddle { +namespace operators { + +class PositiveNegativePairOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE( + ctx->HasInput("Score"), + "Input(Score) of PositiveNegativePairOp should not be null."); + PADDLE_ENFORCE( + ctx->HasInput("Label"), + "Input(Label) of PositiveNegativePairOp should not be null."); + PADDLE_ENFORCE( + ctx->HasInput("QueryID"), + "Input(QueryID) of PositiveNegativePairOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("PositivePair"), + "Output(PositivePair) of PositiveNegativePairOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("NegativePair"), + "Output(NegativePair) of PositiveNegativePairOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("NeutralPair"), + "Output(NeutralPair) of PositiveNegativePairOp should not be null."); + auto scalar_dim = framework::make_ddim({1}); + if (ctx->HasInput("AccumulatePositivePair") || + ctx->HasInput("AccumulateNegativePair") || + ctx->HasInput("AccumulateNeutralPair")) { + PADDLE_ENFORCE(ctx->HasInput("AccumulatePositivePair") && + ctx->HasInput("AccumulateNegativePair") && + ctx->HasInput("AccumulateNeutralPair"), + "All optional inputs(AccumulatePositivePair, " + "AccumulateNegativePair, AccumulateNeutralPair) of " + "PositiveNegativePairOp are required if one of them is " + "specified."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("AccumulatePositivePair"), scalar_dim, + "Shape of AccumulatePositivePair should be {1}."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("AccumulateNegativePair"), scalar_dim, + "Shape of AccumulateNegativePair should be {1}."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("AccumulateNeutralPair"), scalar_dim, + "Shape of AccumulateNeutralPair should be {1}."); + } + + auto score_dim = ctx->GetInputDim("Score"); + auto label_dim = ctx->GetInputDim("Label"); + auto query_dim = ctx->GetInputDim("QueryID"); + PADDLE_ENFORCE_EQ(score_dim.size(), 2, "Score should be a 2-D tensor."); + PADDLE_ENFORCE_EQ(label_dim.size(), 2, "Label should be a 2-D tensor."); + PADDLE_ENFORCE_EQ( + label_dim[0], score_dim[0], + "Tensor Score and Label should have the same height (batch size)."); + PADDLE_ENFORCE_EQ(label_dim[1], 1, + "The width of Label should be 1, i.e. each item should " + "have a scalar label."); + PADDLE_ENFORCE(query_dim == label_dim, + "QueryID should have the same shape as Label."); + if (ctx->HasInput("Weight")) { + PADDLE_ENFORCE(ctx->GetInputDim("Weight") == label_dim, + "Weight should have the same shape as Label."); + } + int column = ctx->Attrs().Get("column"); + auto depth = score_dim[1]; + PADDLE_ENFORCE(column < depth && column >= -depth, + "Attribute column should be in the range of [-%l, %l)", + depth, depth); + + ctx->SetOutputDim("PositivePair", scalar_dim); + ctx->SetOutputDim("NegativePair", scalar_dim); + ctx->SetOutputDim("NeutralPair", scalar_dim); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Score")->type()), + ctx.device_context()); + } +}; + +class PositiveNegativePairOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PositiveNegativePairOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Score", + "(Tensor, float) Model Score on an item (with " + "respect to QueryID). It's a 2-D tensor with shape [batch_size, " + "depth], where the column specified by the attribute \"column\" " + "is used as item score."); + AddInput("Label", + "(Tensor, float) Label of an item (with repsect to " + "QueryId). It's a 2-D tensor with shape [batch_size, 1]."); + AddInput("QueryID", + "(Tensor, int64) Query ID that indicates the context. Its shape " + "should be the same as Label."); + AddInput( + "AccumulatePositivePair", + "(float) Optional. The accumulated number of positive pairs over a " + "stream of data. If provided, the output PositivePair will be " + "initialized with this number rather than 0. it won't be modified " + "in place.") + .AsDispensable(); + AddInput( + "AccumulateNegativePair", + "(float) Optional. The accumulated number of negative pairs over a " + "stream of data. If provided, the output NegativePair will be " + "initialized with this number rather than 0. it won't be modified " + "in place.") + .AsDispensable(); + AddInput("AccumulateNeutralPair", + "(float) Optional. The accumulated number of neutral pairs over a " + "stream of data. If provided, the output NeutralPair will be " + "initialized with this number rather than 0. it won't be modified " + "in place.") + .AsDispensable(); + AddInput("Weight", + "(float) Optional. Weight of current item. If specified, its " + "shape should be the same as Label, and the meaning of the output " + "changes from numbers of pairs to the total sum of pairs' " + "weights. Weight of a pair of items is the average of their " + "weights.") + .AsDispensable(); + AddOutput("PositivePair", + "(float) Number of positive pairs, i.e. the pairs of " + "items that are ranked correctly."); + AddOutput("NegativePair", + "(float) Number of negative pairs, i.e. the pairs of " + "items that are ranked incorrectly."); + AddOutput("NeutralPair", + "(float) Number of neutral pairs, i.e. the pairs of items " + "that have the same score.") + .AsDispensable(); + AddAttr( + "column", + "(int, default -1) The column position of Score used to rank items in " + "descending order. It must be in the range of [-rank(Score), " + "rank(Score)). " + "If `dim < 0`, the dim to reduce is `rank + dim`. " + "Noting that reducing on the first dim will make the LoD info lost.") + .SetDefault(0); + AddComment(R"DOC( + PositiveNegativePairOp can be used to evaluate Learning To Rank(LTR) + model performance. + Within some context, e.g. the "query", a LTR model generates scores + for a list of items, which gives a partial order of the items. + PositiveNegativePairOp takes a list of reference rank order + (Input("Label")) and the model generated scores (Input(Score)) as + inputs and counts the pairs that ranked correctly and incorrectly. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(positive_negative_pair, + ops::PositiveNegativePairOp, + ops::PositiveNegativePairOpMaker); +REGISTER_OP_CPU_KERNEL( + positive_negative_pair, + ops::PositiveNegativePairKernel, + ops::PositiveNegativePairKernel); diff --git a/paddle/operators/positive_negative_pair_op.h b/paddle/operators/positive_negative_pair_op.h new file mode 100644 index 0000000000000000000000000000000000000000..2efd3777e04c17b27c07bccde524de5785af35fe --- /dev/null +++ b/paddle/operators/positive_negative_pair_op.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/utils/Logging.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using LoDTensor = framework::LoDTensor; + +template +class PositiveNegativePairKernel : public framework::OpKernel { + public: + struct PredictionResult { + PredictionResult(T score, T label, T weight) + : score(score), label(label), weight(weight) {} + T score; + T label; + T weight; + }; + + void Compute(const framework::ExecutionContext& context) const override { + auto score_t = context.Input("Score"); + auto label_t = context.Input("Label"); + auto query_t = context.Input("QueryID"); + auto acc_positive_t = context.Input("AccumulatePositivePair"); + auto acc_negative_t = context.Input("AccumulateNegativePair"); + auto acc_neutral_t = context.Input("AccumulateNeutralPair"); + auto positive_t = context.Output("PositivePair"); + auto negative_t = context.Output("NegativePair"); + auto neutral_t = context.Output("NeutralPair"); + auto weight_t = context.Input("Weight"); + + auto score = score_t->data(); + auto label = label_t->data(); + auto query = query_t->data(); + const T* weight = nullptr; + if (weight_t != nullptr) { + weight = weight_t->data(); + } + T* positive = positive_t->mutable_data(context.GetPlace()); + T* negative = negative_t->mutable_data(context.GetPlace()); + T* neutral = neutral_t->mutable_data(context.GetPlace()); + + auto score_dim = score_t->dims(); + auto batch_size = score_dim[0]; + auto width = score_dim[1]; + auto column = context.Attr("column"); + if (column < 0) { + column += width; + } + + // construct document instances for each query: Query => List[, ...] + std::unordered_map> predictions; + for (auto i = 0; i < batch_size; ++i) { + if (predictions.find(query[i]) == predictions.end()) { + predictions.emplace( + std::make_pair(query[i], std::vector())); + } + predictions[query[i]].emplace_back(score[i * width + column], label[i], + weight_t != nullptr ? weight[i] : 1.0); + } + + // for each query, accumulate pair counts + T pos = 0, neg = 0, neu = 0; + if (acc_positive_t != nullptr && acc_negative_t != nullptr && + acc_neutral_t != nullptr) { + pos = acc_positive_t->data()[0]; + neg = acc_negative_t->data()[0]; + neu = acc_neutral_t->data()[0]; + } + auto evaluate_one_list = [&pos, &neg, + &neu](std::vector vec) { + for (auto ite1 = vec.begin(); ite1 != vec.end(); ++ite1) { + for (auto ite2 = ite1 + 1; ite2 != vec.end(); ++ite2) { + if (ite1->label == ite2->label) { // labels are equal, ignore. + continue; + } + T w = (ite1->weight + ite2->weight) * 0.5; + if (ite1->score == ite2->score) { + neu += w; + } + (ite1->score - ite2->score) * (ite1->label - ite2->label) > 0.0 + ? pos += w + : neg += w; + } + } + }; + for (auto prediction : predictions) { + evaluate_one_list(prediction.second); + } + *positive = pos; + *negative = neg; + *neutral = neu; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/precision_recall_op.cc b/paddle/operators/precision_recall_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..1ace4f2a5935dcb4239526c42599a42d288ff552 --- /dev/null +++ b/paddle/operators/precision_recall_op.cc @@ -0,0 +1,183 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/precision_recall_op.h" + +namespace paddle { +namespace operators { + +class PrecisionRecallOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("MaxProbs"), + "Input(MaxProbs) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Indices"), + "Input(Indices) should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Labels"), + "Input(Labels) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("BatchMetrics"), + "Output(BatchMetrics) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("AccumMetrics"), + "Output(AccumMetrics) should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("AccumStatesInfo"), + "Output(AccumStatesInfo) should not be null."); + + int64_t cls_num = + static_cast(ctx->Attrs().Get("class_number")); + auto max_probs_dims = ctx->GetInputDim("MaxProbs"); + auto labels_dims = ctx->GetInputDim("Labels"); + + PADDLE_ENFORCE_EQ(max_probs_dims[1], 1, + "Each instance contains one max probability, so the " + "shape of Input(MaxProbs) should be [batch_size, 1]."); + PADDLE_ENFORCE_EQ(ctx->GetInputDim("Indices"), max_probs_dims, + "The shape of Input(Indices) should be [batch_size, 1]."); + PADDLE_ENFORCE_EQ(max_probs_dims[0], labels_dims[0], + "The 1st dimension of Input(MaxProbs) and " + "Input(Labels) both are batch_size and the shape should " + "be the same."); + PADDLE_ENFORCE_EQ(labels_dims[1], 1, + "The 2nd dimension of Input(Labels) contains instance " + "label and the shape should be equal to 1."); + if (ctx->HasInput("Weights")) { + auto weights_dims = ctx->GetInputDim("Weights"); + PADDLE_ENFORCE_EQ(weights_dims, + framework::make_ddim({max_probs_dims[0], 1}), + "The shape of Input(Weights) should be " + "[batch_size, 1]."); + } + if (ctx->HasInput("StatesInfo")) { + auto states_dims = ctx->GetInputDim("StatesInfo"); + PADDLE_ENFORCE_EQ(states_dims, framework::make_ddim({cls_num, 4}), + "The shape of Input(StatesInfo) should be " + "[class_number, 4]."); + } + + // Layouts of BatchMetrics and AccumMetrics both are: + // [ + // macro average precision, macro average recall, macro average F1 score, + // micro average precision, micro average recall, micro average F1 score + // ] + ctx->SetOutputDim("BatchMetrics", {6}); + ctx->SetOutputDim("AccumMetrics", {6}); + // Shape of AccumStatesInfo is [class_number, 4] + // The layout of each row is: + // [ TP, FP, TN, FN ] + ctx->SetOutputDim("AccumStatesInfo", {cls_num, 4}); + } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("MaxProbs")->type()), + ctx.device_context()); + } +}; + +class PrecisionRecallOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PrecisionRecallOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("MaxProbs", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each row contains the max probability " + "of an instance which computed by the previous top_k (k=1) " + "operator."); + AddInput("Indices", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each row contains the corresponding " + "index which computed by the previous top_k (k=1) operator."); + AddInput("Labels", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. Each element is a label and the " + "value should be in [0, class_number - 1]."); + AddInput("Weights", + "(Tensor, default Tensor) A 2-D tensor with shape N x 1, " + "where N is the batch size. This input is optional. If provided, " + "weight of instance would be considered when computing metrics.") + .AsDispensable(); + AddInput("StatesInfo", + "(Tensor, default Tensor) A 2-D tensor with shape D x 4, " + "where D is the number of classes. This input is optional. If " + "provided, current state will be accumulated to this state and " + "the accumulation state will be the output state.") + .AsDispensable(); + AddOutput("BatchMetrics", + "(Tensor, default Tensor) A 1-D tensor with shape {6}. " + "This output tensor contains metrics for current batch data. " + "The layout is [macro average precision, macro average recall, " + "macro f1 score, micro average precision, micro average recall, " + "micro f1 score]."); + AddOutput("AccumMetrics", + "(Tensor, default Tensor) A 1-D tensor with shape {6}. " + "This output tensor contains metrics for accumulated data. " + "The layout is [macro average precision, macro average recall, " + "macro f1 score, micro average precision, micro average recall, " + "micro f1 score]."); + AddOutput("AccumStatesInfo", + "(Tensor, default Tensor) A 2-D tensor with shape D x 4, " + "where D is equal to class number. This output tensor contains " + "accumulated state variables used to compute metrics. The layout " + "for each class is [true positives, false positives, " + "true negatives, false negatives]."); + AddAttr("class_number", "(int) Number of classes to be evaluated."); + AddComment(R"DOC( +Precision Recall Operator. + +When given Input(Indices) and Input(Labels), this operator can be used +to compute various metrics including: +1. macro average precision +2. macro average recall +3. macro f1 score +4. micro average precision +5. micro average recall +6. micro f1 score + +To compute the above metrics, we need to do statistics for true positives, +false positives and false negatives. Here the count of true negatives is not +necessary, but counting it may provide potential usage and the cost is +trivial, so the operator also provides the count of true negatives. + +We define state as a 2-D tensor with shape [class_number, 4]. Each row of a +state contains statistic variables for corresponding class. Layout of each row +is: TP(true positives), FP(false positives), TN(true negatives), +FN(false negatives). If Input(Weights) is provided, TP, FP, TN, FN will be +calculated by given weight instead of the instance count. + +This operator also supports metrics computing for cross-batch situation. To +achieve this, Input(StatesInfo) should be provided. State of current batch +data will be accumulated to Input(StatesInfo) and Output(AccumStatesInfo) +is the accumulation state. + +Output(BatchMetrics) is metrics of current batch data while +Output(AccumStatesInfo) is metrics of accumulation data. + +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(precision_recall, ops::PrecisionRecallOp, + ops::PrecisionRecallOpMaker); +REGISTER_OP_CPU_KERNEL( + precision_recall, + ops::PrecisionRecallKernel, + ops::PrecisionRecallKernel); diff --git a/paddle/operators/precision_recall_op.h b/paddle/operators/precision_recall_op.h new file mode 100644 index 0000000000000000000000000000000000000000..4a871ce6741469cf9af409ec90215f721d52f36c --- /dev/null +++ b/paddle/operators/precision_recall_op.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +enum StateVariable { TP = 0, FP, TN, FN }; + +template +class PrecisionRecallKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* in0 = ctx.Input("Indices"); + auto* in1 = ctx.Input("Labels"); + auto* in2 = ctx.Input("Weights"); + auto* in3 = ctx.Input("StatesInfo"); + auto* out0 = ctx.Output("BatchMetrics"); + auto* out1 = ctx.Output("AccumMetrics"); + auto* out2 = ctx.Output("AccumStatesInfo"); + + const int* ids_data = in0->data(); + const int* labels_data = in1->data(); + size_t cls_num = static_cast(ctx.Attr("class_number")); + const T* weights_data = in2 ? in2->data() : nullptr; + const T* states_data = in3 ? in3->data() : nullptr; + double* batch_metrics_data = out0->mutable_data(ctx.GetPlace()); + double* accum_metrics_data = out1->mutable_data(ctx.GetPlace()); + out2->mutable_data(ctx.GetPlace()); + auto accum_states = EigenMatrix::From(*out2); + accum_states.setZero(); + T* accum_states_data = out2->data(); + + size_t sample_num = in0->dims()[0]; + size_t state_var_num = 4; // TP FP TN FN + + // get states info for current batch + for (size_t i = 0; i < sample_num; ++i) { + size_t idx = ids_data[i]; + size_t label = labels_data[i]; + + PADDLE_ENFORCE(idx >= 0 && idx < cls_num, + "Class index of each instance should be in " + "[0, class_number)."); + PADDLE_ENFORCE(label >= 0 && label < cls_num, + "Label of each instance should be in [0, class_number)."); + + T w = weights_data ? weights_data[i] : 1.0; + if (idx == label) { + accum_states_data[idx * state_var_num + TP] += w; + for (size_t j = 0; j < cls_num; ++j) { + accum_states_data[j * state_var_num + TN] += w; + } + accum_states_data[idx * state_var_num + TN] -= w; + } else { + accum_states_data[label * state_var_num + FN] += w; + accum_states_data[idx * state_var_num + FP] += w; + for (size_t j = 0; j < cls_num; ++j) { + accum_states_data[j * state_var_num + TN] += w; + } + accum_states_data[idx * state_var_num + TN] -= w; + accum_states_data[label * state_var_num + TN] -= w; + } + } + + ComputeMetrics(accum_states_data, batch_metrics_data, state_var_num, + cls_num); + + if (states_data) { + for (size_t i = 0; i < cls_num; ++i) { + for (size_t j = 0; j < state_var_num; ++j) { + size_t idx = i * state_var_num + j; + accum_states_data[idx] += states_data[idx]; + } + } + } + + ComputeMetrics(accum_states_data, accum_metrics_data, state_var_num, + cls_num); + } + + // expose to be reused + static inline T CalcPrecision(T tp_count, T fp_count) { + if (tp_count > 0.0 || fp_count > 0.0) { + return tp_count / (tp_count + fp_count); + } + return 1.0; + } + + static inline T CalcRecall(T tp_count, T fn_count) { + if (tp_count > 0.0 || fn_count > 0.0) { + return tp_count / (tp_count + fn_count); + } + return 1.0; + } + + static inline T CalcF1Score(T precision, T recall) { + if (precision > 0.0 || recall > 0.0) { + return 2 * precision * recall / (precision + recall); + } + return 0.0; + } + + protected: + void ComputeMetrics(const T* states_data, double* metrics_data, + size_t state_var_num, size_t cls_num) const { + T total_tp_count = 0; + T total_fp_count = 0; + T total_fn_count = 0; + T macro_avg_precision = 0.0; + T macro_avg_recall = 0.0; + + for (size_t i = 0; i < cls_num; ++i) { + T tp_count = states_data[i * state_var_num + TP]; + T fp_count = states_data[i * state_var_num + FP]; + T fn_count = states_data[i * state_var_num + FN]; + total_tp_count += tp_count; + total_fp_count += fp_count; + total_fn_count += fn_count; + macro_avg_precision += CalcPrecision(tp_count, fp_count); + macro_avg_recall += CalcRecall(tp_count, fn_count); + } + macro_avg_precision /= cls_num; + macro_avg_recall /= cls_num; + T macro_f1_score = CalcF1Score(macro_avg_precision, macro_avg_recall); + + T micro_avg_precision = CalcPrecision(total_tp_count, total_fp_count); + T micro_avg_recall = CalcRecall(total_tp_count, total_fn_count); + T micro_f1_score = CalcF1Score(micro_avg_precision, micro_avg_recall); + + // fill metrics data + metrics_data[0] = macro_avg_precision; + metrics_data[1] = macro_avg_recall; + metrics_data[2] = macro_f1_score; + metrics_data[3] = micro_avg_precision; + metrics_data[4] = micro_avg_recall; + metrics_data[5] = micro_f1_score; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc index eef2e34eaacf59b9adacb343e9a0091ebabeaea3..055c471b4561e5fd3c7a65c6f81d66cdce1a5578 100644 --- a/paddle/operators/prelu_op.cc +++ b/paddle/operators/prelu_op.cc @@ -41,17 +41,24 @@ class PReluOpMaker : public framework::OpProtoAndCheckerMaker { PReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input tensor of prelu operator."); - AddInput("Alpha", "The alpha weight of PRelu operator."); - AddOutput("Out", "The output tensor of PRelu operator."); - AddComment(R"DOC(PRelu operator + AddInput("Alpha", "The alpha weight of prelu operator."); + AddOutput("Out", "The output tensor of prelu operator."); + AddComment(R"DOC( +PRelu Operator. The equation is: - f(x) = alpha * x , for x < 0 - f(x) = x , for x >= 0 +$$ +f(x) = +\begin{cases} +\alpha * x, \quad \text{if} \ x < 0 \\ +x, \qquad \text{if} \ x >= 0 +\end{cases} +$$ The input `X` can carry the LoD (Level of Details) information, -or not. And the output shares the LoD with input `X`. +or not. And the output shares the LoD information with input `X`. + )DOC"); } }; diff --git a/paddle/operators/proximal_adagrad_op.cc b/paddle/operators/proximal_adagrad_op.cc index 39fbf800031cd559a49654667e5a6f634384523d..36e460103ab46bf6f1408840a0699793e2be134d 100644 --- a/paddle/operators/proximal_adagrad_op.cc +++ b/paddle/operators/proximal_adagrad_op.cc @@ -83,22 +83,26 @@ class ProximalAdagradOpMaker : public framework::OpProtoAndCheckerMaker { "L1 regularization strength.") .SetDefault(0.0f); AddAttr("l2", - "(float, default 0.0)" + "(float, default 0.0) " "L2 regularization strength.") .SetDefault(0.0f); AddComment(R"DOC( +Proximal Adagrad Optimizer. -Optimizer that implements the proximal adagrad algorithm. +Optimizer that implements the proximal adagrad algorithm: -moment = moment + grad * grad -prox_param = param - learning_rate * grad * (1 / sqrt(moment)) -param = sign(prox_param) / (1 + learning_rate * l2) * - max { |prox_param| - learning_rate * l1 , 0 } +$$ +moment = moment + grad * grad \\ +prox\_param = param - learning\_rate * grad * (1 / \sqrt{moment}) \\ +param = sign(prox\_param) / (1 + learning\_rate * l2) * + \max(|prox\_param| - learning\_rate * l1 , 0) +$$ The paper that proposed Proximal GD: (http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf) Here, we use the adagrad learning rate as specified here: (http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) + )DOC"); } }; diff --git a/paddle/operators/proximal_gd_op.cc b/paddle/operators/proximal_gd_op.cc index e4b014b9f5866ec0791cba9b3998b1734066eeeb..5693d0ec9ebf4c470dfa5141b6eeee431f24f2ea 100644 --- a/paddle/operators/proximal_gd_op.cc +++ b/paddle/operators/proximal_gd_op.cc @@ -67,19 +67,23 @@ class ProximalGDOpMaker : public framework::OpProtoAndCheckerMaker { "L1 regularization strength.") .SetDefault(0.0f); AddAttr("l2", - "(float, default 0.0)" + "(float, default 0.0) " "L2 regularization strength.") .SetDefault(0.0f); AddComment(R"DOC( +ProximalGD Operator. -Optimizer that implements the proximal gradient descent algorithm. +Optimizer that implements the proximal gradient descent algorithm: -prox_param = param - learning_rate * grad -param = sign(prox_param) / (1 + learning_rate * l2) * - max { |prox_param| - learning_rate * l1 , 0 } +$$ +prox\_param = param - learning\_rate * grad \\ +param = sign(prox\_param) / (1 + learning\_rate * l2) * + \max(|prox\_param| - learning\_rate * l1, 0) +$$ The paper that proposed Proximal Gradient Descent: (http://papers.nips.cc/paper/3793-efficient-learning-using-forward-backward-splitting.pdf) + )DOC"); } }; diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc index 17ef2b1d01bd37abf2ece97ed0a307c2f1bf7e6f..061e82412ea5f4f17fd26a7094e68b97138cc09c 100644 --- a/paddle/operators/rank_loss_op.cc +++ b/paddle/operators/rank_loss_op.cc @@ -26,9 +26,9 @@ class RankLossOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext *ctx) const override { // input check - PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null"); - PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null"); - PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null"); + PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Left"), "Input(Left) shouldn't be null."); + PADDLE_ENFORCE(ctx->HasInput("Right"), "Input(Right) shouldn't be null."); auto label_dims = ctx->GetInputDim("Label"); auto left_dims = ctx->GetInputDim("Left"); @@ -50,32 +50,32 @@ class RankLossOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Label", "The label indicating A ranked higher than B or not, row vector."); AddInput("Left", "The output of RankNet for doc A, vector."); - AddInput("Right", "The output of RankNet for doc B, vetor"); + AddInput("Right", "The output of RankNet for doc B, vetor."); AddOutput("Out", "The output loss of RankLoss operator, vector."); - AddComment(R"DOC(RankLoss operator + AddComment(R"DOC( +RankLoss Operator. -Rank loss operator for RankNet[1]. RankNet is a pairwise ranking model with +RankLoss operator for RankNet +(http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf). +RankNet is a pairwise ranking model with one training sample consisting of a pair of doc A and B, and the label P indicating that A is ranked higher than B or not: P = {0, 1} or {0, 0.5, 1}, where 0.5 means no information about the rank of the input pair. -The RankLoss operator contains three inputs: Left (o_i), Right (o_j) and Label -(P_{i,j}), which represent the output of RankNet for two docs and the label -respectively, and yields the rank loss C_{i,j} by following the expression +The RankLoss operator takes three inputs: Left (o_i), Right (o_j) and Label +(P_{i,j}), which represent the output of RankNet for the two docs and the label, +respectively, and yields the rank loss C_{i,j} using the following equation: -\f[ +\f$$ C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + log(1 + e^{o_{i,j}}) \\ o_{i,j} = o_i - o_j \\ \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} -\f] +\f$$ The operator can take inputs of one sample or in batch. -[1]. Chris Burges, Tal Shaked, Erin Renshaw, et al. Learning to - Rank using Gradient Descent. - http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf )DOC"); } }; diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 40303e3adf4db7e8336ed72667fe69afa56c3f69..0075ccd24271bf83f139e121efad00c2316cc11b 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -12,181 +12,612 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/operators/recurrent_op.h" - -#include -#include - +#include +#include "paddle/framework/executor.h" #include "paddle/framework/op_registry.h" -#include "paddle/operators/net_op.h" namespace paddle { namespace operators { +constexpr char kInputs[] = "inputs"; +constexpr char kInitialStates[] = "initial_states"; +constexpr char kParameters[] = "parameters"; +constexpr char kOutputs[] = "outputs"; +constexpr char kStepScopes[] = "step_scopes"; +constexpr char kExStates[] = "ex_states"; +constexpr char kStates[] = "states"; +constexpr char kStepBlock[] = "step_block"; +constexpr char kReverse[] = "reverse"; +constexpr char kIsTrain[] = "is_train"; +#define GRAD_SUFFIX "@GRAD" +constexpr char kInputGrads[] = "inputs" GRAD_SUFFIX; +constexpr char kOutputGrads[] = "outputs" GRAD_SUFFIX; +constexpr char kParamGrads[] = "parameters" GRAD_SUFFIX; +constexpr char kInitStateGrads[] = "initial_states" GRAD_SUFFIX; -using Scope = framework::Scope; -using Variable = framework::Variable; -using Tensor = framework::Tensor; -using LoDTensor = framework::LoDTensor; - -void RecurrentAlgorithm::Run(const Scope& scope, - const platform::DeviceContext& dev_ctx) const { - auto* input0 = scope.FindVar(arg_->inlinks[0]); - PADDLE_ENFORCE_NOT_NULL(input0); - size_t seq_len = input0->GetMutable()->dims()[0]; - PADDLE_ENFORCE_GT(seq_len, 0); - - CreateScopes(scope, seq_len); - auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); - InitMemories(step_scopes[0]); - - for (size_t step_id = 0; step_id < seq_len; step_id++) { - if (step_id > 0) { - rnn::LinkMemories(step_scopes, arg_->states, step_id, -1); +using StepScopeVar = std::vector; + +// StepScopes manages scopes inside RNN. +// StepScopes::CurScope() get the current scope +// StepScopes::ExScope() get the ex-scope, or scope in previous time step. +// StepScopes::Next() move to next time step. +// +// if is_train = False, then +// there are two scopes for the RNN and just support forward. +// else +// the len(scopes) == seq_len +// +// if is_backward = True, then +// reversely access scopes +// else +// access scopes from begin to end. +class StepScopes { + public: + StepScopes(const framework::Scope &parent, StepScopeVar *scopes, + bool is_train, size_t seq_len, bool is_backward = false) + : counter_(is_backward ? seq_len - 1 : 0UL), + scopes_(scopes), + is_train_(is_train), + is_backward_(is_backward) { + size_t num_step_scopes = is_train ? seq_len : 2; + PADDLE_ENFORCE(is_train || !is_backward, + "Cannot backward when is not training"); + if (!is_backward_) { + PADDLE_ENFORCE(scopes->empty()); + scopes->reserve(static_cast(num_step_scopes)); + for (size_t i = 0; i < num_step_scopes; ++i) { + scopes->emplace_back(&parent.NewScope()); + } } - (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx); -} - -void RecurrentAlgorithm::CreateScopes(const Scope& scope, - size_t seq_len) const { - // TODO(superjom) Only two scopes are needed for inference, this case will be - // supported later. - auto* step_scopes_var = scope.FindVar(arg_->step_scopes); - PADDLE_ENFORCE(step_scopes_var != nullptr, ""); - auto* step_scopes = step_scopes_var->GetMutable>(); - - // Now all variables in scope must be created outside of op. - PADDLE_ENFORCE_NOT_NULL(stepnet_); - PADDLE_ENFORCE(!(*stepnet_)->Outputs().empty(), - "step_unit_ op has no outputs"); - - if (seq_len > step_scopes->size()) { - for (size_t i = step_scopes->size(); i < seq_len; ++i) { - auto& step_scope = scope.NewScope(); - - // create step net's temp inputs - for (auto& input : (*stepnet_)->Inputs()) { - // the weight are located in parent scope - for (auto& var_name : input.second) { - if (!step_scope.FindVar(var_name)) { - step_scope.Var(var_name)->GetMutable(); - } + } + + framework::Scope &CurScope() { return GetScope(counter_); } + + framework::Scope &ExScope() { + auto &scope = GetScope(is_backward_ ? counter_ + 1 : counter_ - 1); + return scope; + } + + void Next() { + if (is_backward_) { + --counter_; + } else { + ++counter_; + } + } + + private: + framework::Scope &GetScope(size_t scope_id) const { + if (!is_train_) { + scope_id %= 2; + } + PADDLE_ENFORCE_LT(scope_id, scopes_->size()); + return *(*scopes_)[scope_id]; + } + + size_t counter_; + StepScopeVar *scopes_; + bool is_train_; + bool is_backward_; +}; + +// Base class for RecurrentOp/RecurrentGradOp +// Some common protected functions for RecurrentOp/RecurrentGradOp +class RecurrentBase : public framework::OperatorBase { + public: + RecurrentBase(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + + protected: + // Get SequenceLength from Scope + // The sequence length is got from input tensor. The input tensor's + // dimension should be [SEQ_LEN, ..., ...]. The first of the tensor's shape + // is SEQ_LEN. The second of the tensor's shape could be the batch size or + // nested sequence length. + int64_t GetSequenceLength(const framework::Scope &scope) const { + // Dim format SEQ_LEN, BATCH_SIZE, ... + int64_t seq_len = -1; + auto &all_inputs = Inputs(kInputs); + PADDLE_ENFORCE(!all_inputs.empty()); + for (auto &iname : all_inputs) { + auto *var = scope.FindVar(iname); + PADDLE_ENFORCE(var != nullptr); + PADDLE_ENFORCE(var->IsType()); + auto &dim = var->Get().dims(); + if (seq_len == -1) { + seq_len = dim[0]; + } else { + PADDLE_ENFORCE_EQ(seq_len, dim[0]); + } + } + return seq_len; + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.Var, dst_vars)): + // dst_tensor.ShareDataWith(src_tensor) + static void LinkTensor(const framework::Scope &src_scope, + const std::vector &src_vars, + framework::Scope *dst_scope, + const std::vector &dst_vars) { + LinkTensorWithCallback( + src_scope, src_vars, dst_scope, dst_vars, + [&](const framework::Tensor &src, framework::Tensor *dst) { + dst->ShareDataWith(src); + }); + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.Var, dst_vars)): + // callback(src_tensor, &dst_tensor) + template + static void LinkTensorWithCallback(const framework::Scope &src_scope, + const std::vector &src_vars, + framework::Scope *dst_scope, + const std::vector &dst_vars, + Callback callback) { + PADDLE_ENFORCE_EQ(src_vars.size(), dst_vars.size()); + for (size_t i = 0; i < dst_vars.size(); ++i) { + VLOG(10) << "Link " << src_vars[i] << " to " << dst_vars[i]; + AccessTensor(src_scope, src_vars[i], dst_scope, dst_vars[i], callback); + } + } + + // for src_tensor, dst_tensor in zip(map(src_scope.FindVar, src_vars), + // map(dst_scope.FindVar, dst_vars)): + // callback(src_tensor, &dst_tensor) + template + static void LinkTensorWithCallback(const framework::Scope &src_scope, + const std::vector &src_vars, + const framework::Scope &dst_scope, + const std::vector &dst_vars, + Callback callback) { + PADDLE_ENFORCE_EQ(src_vars.size(), dst_vars.size()); + for (size_t i = 0; i < dst_vars.size(); ++i) { + VLOG(10) << "Link " << src_vars[i] << " to " << dst_vars[i]; + AccessTensor(src_scope, src_vars[i], dst_scope, dst_vars[i], callback); + } + } + + // (seq_len, shape) -> return [seq_len] + list(shape) + static framework::DDim PrependDims(size_t seq_len, + const framework::DDim &src) { + auto dims = framework::vectorize(src); + dims.insert(dims.begin(), static_cast(seq_len)); + return framework::make_ddim(dims); + } + + private: + template + static void AccessTensor(const framework::Scope &src_scope, + const std::string &src_var_name, + framework::Scope *dst_scope, + const std::string &dst_var_name, Callback callback) { + auto *src_var = src_scope.FindVar(src_var_name); + PADDLE_ENFORCE(src_var != nullptr); + auto &src_tensor = src_var->Get(); + + auto *dst_var = dst_scope->Var(dst_var_name); + auto *dst_tensor = dst_var->GetMutable(); + callback(src_tensor, dst_tensor); + } + + template + static void AccessTensor(const framework::Scope &src_scope, + const std::string &src_var_name, + const framework::Scope &dst_scope, + const std::string &dst_var_name, Callback callback) { + auto *src_var = src_scope.FindVar(src_var_name); + PADDLE_ENFORCE(src_var != nullptr); + auto &src_tensor = src_var->Get(); + auto *dst_var = dst_scope.FindVar(dst_var_name); + PADDLE_ENFORCE(dst_var != nullptr); + auto *dst_tensor = dst_var->GetMutable(); + callback(src_tensor, dst_tensor); + } +}; + +class RecurrentOp : public RecurrentBase { + public: + RecurrentOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : RecurrentBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto seq_len = static_cast(this->GetSequenceLength(scope)); + VLOG(3) << "Static RNN input sequence length = " << seq_len; + StepScopes scopes = CreateStepScopes(scope, seq_len); + auto reverse = Attr(kReverse); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + for (size_t i = 0; i < seq_len; ++i) { + size_t seq_offset = reverse ? seq_len - i - 1 : i; + VLOG(3) << "Recurrent operate at the time step " << seq_offset; + + auto &cur_scope = scopes.CurScope(); + + // Link outside::input --> inside::input + // inside::input = outside::input[seq_offset: seq_offset+1] + LinkTensorWithCallback( + scope, Inputs(kInputs), &cur_scope, Inputs(kInputs), + [&seq_offset](const framework::Tensor &outside, + framework::Tensor *inside) { + inside->ShareDataWith(outside.Slice(seq_offset, seq_offset + 1)); + auto dims = framework::vectorize(inside->dims()); + dims.erase(dims.begin()); + inside->Resize(framework::make_ddim(dims)); + }); + + if (i == 0) { + // Link initial states --> ex_states + LinkTensor(scope, Inputs(kInitialStates), &cur_scope, + Attr>(kExStates)); + } else { + auto &ex_scope = scopes.ExScope(); + // Link ex_scope::state --> cur_scope::ex_state + LinkTensor(ex_scope, Attr>(kStates), + &cur_scope, Attr>(kExStates)); + } + + // Every inputs are linked now, execute! + executor.Run(*program, &cur_scope, block->ID(), + false /*create_local_scope*/); + + // Copy inside::output -> outside::output + // outside::output[seq_offset: seq_offset + 1] = inside::output + this->LinkTensorWithCallback( + cur_scope, Outputs(kOutputs), scope, Outputs(kOutputs), + [&](const framework::LoDTensor &src_tensor, + framework::LoDTensor *dst_tensor) { + if (i == 0) { // create output tensor at begin + dst_tensor->Resize(PrependDims(seq_len, src_tensor.dims())); + dst_tensor->mutable_data(dev_ctx.GetPlace(), src_tensor.type()); + } + + auto dst_out = dst_tensor->Slice(seq_offset, seq_offset + 1); + // Explicit copy output since the local RNN scope can be destroyed + // early. + dst_out.CopyFrom(src_tensor, dev_ctx.GetPlace(), dev_ctx); + }); + + scopes.Next(); + } + } + + private: + StepScopes CreateStepScopes(const framework::Scope &scope, + size_t seq_len) const { + auto *var = scope.FindVar(Output(kStepScopes)); + PADDLE_ENFORCE(var != nullptr); + return StepScopes(scope, var->GetMutable(), + Attr(kIsTrain), seq_len); + } +}; + +class RecurrentGradOp : public RecurrentBase { + public: + RecurrentGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : RecurrentBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto seq_len = static_cast(GetSequenceLength(scope)); + StepScopes scopes = CreateStepScopes(scope, seq_len); + auto reverse = Attr(kReverse); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + for (size_t step_id = 0; step_id < seq_len; ++step_id) { + size_t seq_offset = reverse ? step_id : seq_len - step_id - 1; + VLOG(3) << "Recurrent backward operate at the time step " << seq_offset; + auto &cur_scope = scopes.CurScope(); + // Link outside::output_grads --> inside::output_grads + // inside::output_grad = outside::output_grad[seq_offset:seq_offset+1] + LinkTensorWithCallback( + scope, Inputs(kOutputGrads), &cur_scope, Inputs(kOutputGrads), + [&](const framework::Tensor &outside, framework::Tensor *inside) { + inside->ShareDataWith(outside.Slice(seq_offset, seq_offset + 1)); + auto dims = framework::vectorize(inside->dims()); + dims.erase(dims.begin()); + inside->Resize(framework::make_ddim(dims)); + }); + auto og_set = List2Set(Inputs(kOutputGrads)); + + if (VLOG_IS_ON(10)) { + std::ostringstream sout; + std::copy(og_set.begin(), og_set.end(), + std::ostream_iterator(sout, ",")); + VLOG(10) << " RNN output gradients = [" << sout.str() << "]"; + } + + // Link states + // if cur_scope::cur_state_grad in out_grads: + // cur_scope::cur_state_grad += ex_scope::ex_state_grad + // else: + // ex_scope::ex_state_grad --> cur_scope::cur_state_grad + if (step_id != 0) { // not at beginning + auto &ex_scope = scopes.ExScope(); + auto ex_state_grads = + GradVarLists(Attr>(kExStates)); + auto cur_state_grads = + GradVarLists(Attr>(kStates)); + + PADDLE_ENFORCE_EQ(ex_state_grads.size(), cur_state_grads.size()); + for (size_t i = 0; i < ex_state_grads.size(); ++i) { + auto &cur_grad = cur_state_grads[i]; + auto &ex_grad = ex_state_grads[i]; + auto &ex_tensor = + ex_scope.FindVar(ex_grad)->Get(); + + VLOG(10) << " RNN link " << cur_grad << " from " << ex_grad; + auto *cur_grad_var = cur_scope.Var(cur_grad); + auto cur_grad_tensor = + cur_grad_var->GetMutable(); + cur_grad_tensor->CopyFrom(ex_tensor, dev_ctx.GetPlace(), dev_ctx); } } - // create stepnet's outputs - for (const auto& output : (*stepnet_)->Outputs()) { - for (auto& var_name : output.second) { - step_scope.Var(var_name); + + VLOG(5) << "Recurrent memory linking finished "; + // Run step block with cur_scope + executor.Run(*program, &cur_scope, block->ID(), + false /*create_local_scope*/); + + VLOG(5) << "executor.Run finished "; + + auto local_var_names = LocalVarNames(cur_scope); + + // Accumulate params + // if (step == 0): + // outside::param_grad = 0.0 + // outside::param_grad += inside::param_grad + { + auto &pg_names = Outputs(kParamGrads); + auto &p_names = Inputs(kParameters); + PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); + + for (size_t param_id = 0; param_id < pg_names.size(); ++param_id) { + auto inside_grad_name = framework::GradVarName(p_names[param_id]); + + // If does not compute gradient of that variable inside rnn, just + // continue + if (local_var_names.find(inside_grad_name) == local_var_names.end()) { + continue; + } + + // zero gradient variable in step 0 + if (step_id == 0) { + auto &inside_tensor = cur_scope.FindVar(inside_grad_name) + ->Get(); + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(inside_tensor.type()); + attrs["shape"] = framework::vectorize2int(inside_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {pg_names[param_id]}}}, attrs); + zero_op->Run(scope, dev_ctx); + } + + auto new_inside_name = cur_scope.Rename(inside_grad_name); + // sum gradient + + auto sum_op = framework::OpRegistry::CreateOp( + "sum", {{"X", {pg_names[param_id], new_inside_name}}}, + {{"Out", {pg_names[param_id]}}}, {}); + sum_op->Run(cur_scope, dev_ctx); + + cur_scope.Rename(new_inside_name, inside_grad_name); } } - step_scopes->emplace_back(&step_scope); + VLOG(5) << "Accumulate Parameter finished "; + + // Copy input gradient from inside to outside + // outside::input_grad[seq_offset: seq_offset + 1] = inside::input_grad + LinkTensorWithCallback( + cur_scope, GradVarLists(Inputs(kInputs)), scope, Outputs(kInputGrads), + [&](const framework::LoDTensor &inside, + framework::LoDTensor *outside) { + if (inside.memory_size() == 0) { // IG is not created. + return; + } + if (step_id == 0) { // alloc memory + outside->Resize(PrependDims(seq_len, inside.dims())); + outside->mutable_data(dev_ctx.GetPlace(), inside.type()); + } + + auto dst = outside->Slice(seq_offset, seq_offset + 1); + dst.CopyFrom(inside, dev_ctx.GetPlace(), dev_ctx); + }); + VLOG(5) << "Link outside gradient finished "; + + if (step_id + 1 == seq_len) { // at_end + // copy initialize states gradient from inside to outside + LinkTensorWithCallback( + cur_scope, GradVarLists(Attr>(kExStates)), + scope, Outputs(kInitStateGrads), + [&](const framework::LoDTensor &inside, + framework::LoDTensor *outside) { + outside->Resize(inside.dims()); + outside->mutable_data(dev_ctx.GetPlace(), inside.type()); + outside->CopyFrom(inside, dev_ctx.GetPlace(), dev_ctx); + }); + VLOG(5) << "Link initialize state gradient finished "; + } + scopes.Next(); } } -} - -void RecurrentAlgorithm::InitMemories(Scope* step_scope) const { - for (auto& attr : arg_->states) { - auto* pre_mem = step_scope->Var(attr.pre_var)->GetMutable(); - PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr, - "memory [%s]'s boot variable [%s] not exists", attr.var, - attr.boot_var); - auto* boot_mem = - step_scope->FindVar(attr.boot_var)->GetMutable(); - pre_mem->Resize(boot_mem->dims()); - PADDLE_ENFORCE_EQ(pre_mem->dims().size(), 2); - pre_mem->ShareDataWith(*boot_mem); - } -} - -const rnn::ArgumentName RecurrentOp::kArgName{ - "step_net", "step_scopes", "inputs", "outputs", - "states", "ex_states", "initial_states"}; - -const rnn::ArgumentName RecurrentGradientOp::kArgName{ - "step_net", "step_scopes@GRAD", "outputs@GRAD", "inputs@GRAD", - "states", "ex_states", "initial_states@GRAD"}; - -RecurrentOp::RecurrentOp(const std::string& type, - const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs) - : OperatorBase(type, inputs, outputs, attrs) { - rnn::InitArgument(kArgName, &arg_, *this); - alg_.Init(&arg_, &stepnet_); -} - -class RecurrentAlgorithmProtoAndCheckerMaker - : public framework::OpProtoAndCheckerMaker { + + private: + StepScopes CreateStepScopes(const framework::Scope &scope, + size_t seq_len) const { + auto *var = scope.FindVar(Input(kStepScopes)); + PADDLE_ENFORCE(var != nullptr); + return StepScopes(scope, var->GetMutable(), + Attr(kIsTrain), seq_len, true /*is_backward*/); + } + + std::unordered_set List2Set( + const std::vector &list) const { + std::unordered_set local_var_name_set; + local_var_name_set.reserve(list.size()); + for (auto &each : list) { + local_var_name_set.insert(each); + } + return local_var_name_set; + } + + std::unordered_set LocalVarNames( + const framework::Scope &scope) const { + return this->List2Set(scope.GetAllNames(false)); + } + static std::vector GradVarLists( + const std::vector &var_names) { + std::vector retv; + retv.reserve(var_names.size()); + std::transform(var_names.begin(), var_names.end(), std::back_inserter(retv), + framework::GradVarName); + return retv; + } +}; + +class RecurrentOpProtoMaker : public framework::OpProtoAndCheckerMaker { public: - RecurrentAlgorithmProtoAndCheckerMaker(framework::OpProto* proto, - framework::OpAttrChecker* op_checker) + RecurrentOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - const auto& name = RecurrentOp::kArgName; - // inputs and outputs stored in proto - AddInput(name.inlinks, - "the inputs that need to be segmented for each step.") + AddInput(kInputs, "rnn inputs").AsDuplicable(); + AddInput(kInitialStates, "rnn initial states").AsDuplicable(); + AddInput(kParameters, + "Parameters are used by step block as its input. However, the " + "input is not a sequence tensor. Every time step, each operator " + "in step block just use the parameter directly.") .AsDuplicable(); - AddInput(name.initial_states, "variables to initialize states.") + AddOutput(kOutputs, + "The output sequence of RNN. The sequence length must be same.") .AsDuplicable(); + AddOutput(kStepScopes, + "StepScopes contain all local variables in each time step."); + AddAttr>(kExStates, + string::Sprintf( + R"DOC(The ex-state variable names. +The ex-state means the state value in the ex-timestep or the previous time step +[%s, %s, %s] must be the same order)DOC", + kExStates, kStates, kInitStateGrads)); + AddAttr>( + kStates, + string::Sprintf( + "The state variable names. [%s, %s, %s] must be the same order", + kExStates, kStates, kInitStateGrads)); + AddAttr(kStepBlock, + "The step block inside RNN"); + AddAttr(kReverse, R"DOC(Calculate RNN reversely or not. +By default reverse=False - AddOutput(name.outlinks, "the outputs that need to concated for all steps.") - .AsDuplicable(); - AddOutput(name.step_scopes, "step scopes"); +Assume the input data is [A, B, C, D] + +if reverse is False: + the computation of RNN is like + A B C D + | | | | + v v v v + rnn -----> rnn -----> rnn ----> rnn + | | | | + v v v v + o o o o + +if reverse is True + the computation of RNN is like + A B C D + | | | | + v v v v + rnn <----- rnn <----- rnn <---- rnn + | | | | + v v v v + o o o o +)DOC").SetDefault(false); + AddAttr(kIsTrain, "").SetDefault(true); + AddComment(R"DOC( +Static Length Recurrent Operator. + +The static length recurrent operator can only operate on fixed size sequence +data, i.e. in each mini-batch, the sequence length of all inputs are the same. + +)DOC"); + } +}; + +class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - // Attributes stored in AttributeMap - AddAttr>(name.ex_states, "names of pre-states"); - AddAttr>(name.states, "names of states"); + protected: + virtual std::unique_ptr Apply() const { + auto *grad = new framework::OpDescBind(); + grad->SetType("recurrent_grad"); + for (auto &input_param : this->InputNames()) { + grad->SetInput(input_param, this->Input(input_param)); + grad->SetOutput(framework::GradVarName(input_param), + this->InputGrad(input_param)); + } + + for (auto &output_param : this->OutputNames()) { + if (output_param == kStepScopes) { + grad->SetInput(output_param, this->Output(output_param)); + grad->SetInput(framework::GradVarName(output_param), + this->Output(output_param)); + } else { + grad->SetInput(output_param, this->Output(output_param)); + grad->SetInput(framework::GradVarName(output_param), + this->OutputGrad(output_param)); + } + } + grad->SetAttrMap(this->Attrs()); + grad->SetBlockAttr(kStepBlock, *grad_block_[0]); - AddComment("This is a recurrent group operator."); + return std::unique_ptr(grad); } }; -void RecurrentGradientAlgorithm::Run( - const Scope& scope, const platform::DeviceContext& dev_ctx) const { - auto* input0 = scope.FindVar(arg_->inlinks[0]); - PADDLE_ENFORCE_NOT_NULL(input0); - size_t seq_len = input0->GetMutable()->dims()[0]; - auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); - for (int step_id = seq_len - 1; step_id >= 0; --step_id) { - if (static_cast(step_id) != seq_len - 1) { - rnn::LinkMemories(step_scopes, arg_->states, step_id, 1); +class RecurrentGradOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + std::vector input{kInputs, kInitialStates}; + std::vector output{kOutputs}; + for (auto &s : input) { + PADDLE_ENFORCE(ctx->HasInputs(s)); + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(s))); + } + for (auto &s : output) { + PADDLE_ENFORCE(ctx->HasInputs(s)); + } + for (auto &s : input) { + ctx->SetOutputsDim(framework::GradVarName(s), ctx->GetInputsDim(s)); } - (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx); - LinkBootMemoryGradients(step_scopes[0]); -} - -void RecurrentGradientAlgorithm::LinkBootMemoryGradients( - Scope* step_scope) const { - for (auto& attr : arg_->states) { - PADDLE_ENFORCE(step_scope->FindVar(attr.var) != nullptr, - "memory variable [%s] does not exists", attr.var); - PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr, - "boot variable [%s] does not exists", attr.boot_var); - auto* mem_grad = step_scope->Var(attr.var)->GetMutable(); - auto* boot_mem_grad = - step_scope->Var(attr.boot_var)->GetMutable(); - boot_mem_grad->Resize(mem_grad->dims()); - boot_mem_grad->ShareDataWith(*mem_grad); - } -} - -RecurrentGradientOp::RecurrentGradientOp( - const std::string& type, const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs) - : OperatorBase(type, inputs, outputs, attrs) { - rnn::InitArgument(kArgName, &arg_, *this, true /*is grad*/); - alg_.Init(&arg_, &stepnet_); -} + if (ctx->HasInputs(kParameters)) { + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters))); + ctx->SetOutputsDim(framework::GradVarName(kParameters), + ctx->GetInputsDim(kParameters)); + } + } +}; } // namespace operators } // namespace paddle -REGISTER_OP(recurrent, paddle::operators::RecurrentOp, - paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker, - recurrent_grad, paddle::operators::RecurrentGradientOp); +REGISTER_OPERATOR(recurrent, paddle::operators::RecurrentOp, + paddle::operators::RecurrentOpProtoMaker, + paddle::operators::RecurrentGradOpDescMaker); +REGISTER_OPERATOR(recurrent_grad, paddle::operators::RecurrentGradOp, + paddle::operators::RecurrentGradOpShapeInference); diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h deleted file mode 100644 index 253d7e3284360ceaddce9ef5f8f9a3ea4793d740..0000000000000000000000000000000000000000 --- a/paddle/operators/recurrent_op.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once - -#include "paddle/framework/operator.h" -#include "paddle/operators/net_op.h" -#include "paddle/operators/rnn/recurrent_op_utils.h" - -namespace paddle { -namespace operators { - -// The sequence format in RecurrentOp is Tensor now. -// TODO(Superjom) -// 1. No-padding computing for sequences with indifinite length in one batch. -// 2. Hierarchical RNN for sequence with sub-sequence. -// 3. Internal Memory. -// 4. More Complex RNN architecture, such as Gated Feedback RNN. -// Refer to: https://arxiv.org/pdf/1502.02367.pdf - -class RecurrentAlgorithm { - public: - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const; - - void Init(rnn::Argument* arg, - std::unique_ptr* stepnet) { - PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before."); - arg_ = arg; - stepnet_ = stepnet; - } - - protected: - /* - * The step scopes will be stored in the father scope as a variable. - * - * NOTE the scopes are reused in both the forward and backward, so just - * create once and expand its size if more steps need. - */ - void CreateScopes(const framework::Scope& scope, size_t seq_len) const; - - const std::vector& GetStepScopes( - const framework::Scope& scope) const { - return *scope.FindVar(arg_->step_scopes) - ->GetMutable>(); - } - - void InitMemories(framework::Scope* step_scopes) const; - - private: - std::unique_ptr* stepnet_; - rnn::Argument* arg_; -}; - -class RecurrentGradientAlgorithm { - /** - * RNN's backward alogorithm. - * - * To accelerate the development of RecurrentGradientOp, we decouple RNN's - * algorithm and `OperatorBase`'s implementation, the former contains the core - * implementation of a RNN, and will keep stable even if the framework changes - * a - * lot, and the latter is a wrapper acts like an dapter for it to make RNN an - * operator. - */ - public: - void Init(rnn::Argument* arg, - std::unique_ptr* stepnet) { - PADDLE_ENFORCE_NOT_NULL(stepnet, "stepnet should be set before."); - arg_ = std::move(arg); - stepnet_ = stepnet; - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const; - - void LinkBootMemoryGradients(framework::Scope* step_scopes) const; - - protected: - inline const std::vector& GetStepScopes( - const framework::Scope& scope) const { - return *scope.FindVar(arg_->step_scopes) - ->GetMutable>(); - } - - private: - rnn::Argument* arg_; - std::unique_ptr* stepnet_; -}; - -class RecurrentOp : public framework::OperatorBase { - public: - RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs); - - RecurrentOp(const RecurrentOp& o) - : framework::OperatorBase( - static_cast(o)) { - // TODO(yuyang18): Implement copy ctor well. - PADDLE_THROW("Not implemented"); - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const override { - alg_.Run(scope, dev_ctx); - } - - void set_stepnet(std::unique_ptr net) { - stepnet_ = std::move(net); - } - - const OperatorBase& stepnet() const { return *stepnet_; } - - static const rnn::ArgumentName kArgName; - - private: - RecurrentAlgorithm alg_; - rnn::Argument arg_; - std::unique_ptr stepnet_; -}; - -class RecurrentGradientOp : public framework::OperatorBase { - public: - RecurrentGradientOp(const std::string& type, - const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs); - - RecurrentGradientOp(const RecurrentGradientOp& o) - : framework::OperatorBase( - static_cast(o)) { - // TODO(yuyang18): Implement Copy ctor. - PADDLE_THROW("Not Implemented"); - } - - void Run(const framework::Scope& scope, - const platform::DeviceContext& dev_ctx) const override { - alg_.Run(scope, dev_ctx); - } - - static const rnn::ArgumentName kArgName; - - /* - * set a stepnet that is created according to a RecurrentOp's stepnet. - */ - void set_stepnet(std::unique_ptr net) { - stepnet_ = std::move(net); - } - const OperatorBase& stepnet() const { return *stepnet_; } - - private: - RecurrentGradientAlgorithm alg_; - std::unique_ptr stepnet_; - rnn::Argument arg_; -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/reduce_op.cc b/paddle/operators/reduce_op.cc index 0599daa7688a5658ebea8902c4e15e63570539fb..2589a54cfc7fc5bc11ae983797d480a134e0eb25 100644 --- a/paddle/operators/reduce_op.cc +++ b/paddle/operators/reduce_op.cc @@ -80,24 +80,27 @@ class ReduceOpMaker : public framework::OpProtoAndCheckerMaker { public: ReduceOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "(Tensor) The input tensor. Tensors with rank at most 6 are supported"); + AddInput("X", + "(Tensor) The input tensor. Tensors with rank at most 6 are " + "supported."); AddOutput("Out", "(Tensor) The result tensor."); AddAttr( "dim", - "(int, default 1) The dimension to reduce. " + "(int, default 0) The dimension to reduce. " "Must be in the range [-rank(input), rank(input)). " "If `dim < 0`, the dim to reduce is `rank + dim`. " - "Noting that reducing on the first dim will make the LoD info lost.") + "Note that reducing on the first dim will make the LoD info lost.") .SetDefault(0); AddAttr("keep_dim", "(bool, default false) " "If true, retain the reduced dimension with length 1.") .SetDefault(false); comment_ = R"DOC( -{ReduceOP} operator computes the {reduce} of input tensor along the given dimension. -The result tensor has 1 fewer dimension than the input unless `keep_dim` is true. +{ReduceOp} Operator. + +This operator computes the {reduce} of input tensor along the given dimension. +The result tensor has 1 fewer dimension than the input unless keep_dim is true. + )DOC"; AddComment(comment_); } diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 9213cc7a85822e4c78ef72aec2bf86d2edac023a..ba774ec2160c0460867de42f7ad9d5cd65ad8d6a 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -71,8 +71,11 @@ class ReshapeOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input tensor of reshape operator."); AddOutput("Out", "The output tensor of reshape operator."); - AddAttr>("shape", "Target shape of reshape operator."); - AddComment(R"DOC(Reshape operator + AddAttr>("shape", + "(vector) " + "Target shape of reshape operator."); + AddComment(R"DOC( +Reshape Operator. Reshape Input(X) into the shape specified by Attr(shape). @@ -81,7 +84,7 @@ Given a 2-D tensor X with 2 rows and 2 columns [[1, 2], [3, 4]] -with target shape = [1, 4], the reshape operator will transform +and target shape = [1, 4], the reshape operator will transform the tensor X into a 1-D tensor: [1, 2, 3, 4] diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index fd5567a365c4c843de3b8aec7fa77164f16644a4..a9c45f639c6728ff2fd6de6fcdadfe5032a705d7 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -68,22 +68,22 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("Param", "(Tensor, default Tensor) " - "Input parameter value that has to be updated"); + "Input parameter value that has to be updated."); AddInput("MeanSquare", "(Tensor, default Tensor)" - " The mean square value that gets updated"); + " The mean square value that gets updated."); AddInput("LearningRate", "(Tensor, default Tensor) " - "The learning rate should be a tensor of size 1"); + "The learning rate should be a tensor of size 1."); AddInput("Grad", "(Tensor, default Tensor) " - "Input gradient of the parameter"); + "Input gradient of the parameter."); AddInput("Moment", - "(Tensor, default Tensor) The moment that gets updated"); + "(Tensor, default Tensor) The moment that gets updated."); - AddOutput("ParamOut", "(Tensor) Output updated parameter value"); - AddOutput("MomentOut", "(Tensor) Output updated moment"); - AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value"); + AddOutput("ParamOut", "(Tensor) Output updated parameter value."); + AddOutput("MomentOut", "(Tensor) Output updated moment."); + AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value."); AddAttr("epsilon", "(float, default 1e-10) Constant " @@ -93,18 +93,19 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { "(float, default 0.9) " "Discounting factor for coming gradient.") .SetDefault(0.9f); - AddAttr("momentum", "(float, default 0.0) Constant value") + AddAttr("momentum", "(float, default 0.0) Constant value.") .SetDefault(0.0f); AddComment(R"DOC( +Rmsprop Optimizer. -RMSprop - -MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad +$$ +MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad \\ MomentOut = momentum * Moment + - LearningRate * Grad / sqrt(MeanSquareOut + epsilon) + \frac{LearningRate * Grad}{\sqrt{MeanSquareOut + epsilon}} \\ ParamOut = Param - MomentOut +$$ -The original slides that proposed RMSprop: Slide 29 of +The original slides that proposed Rmsprop: Slide 29 of http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) )DOC"); diff --git a/paddle/operators/rnn_memory_helper_op.cc b/paddle/operators/rnn_memory_helper_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..b621c7f1ba3f9e9613dea5bc98ef74c7c6dae9a0 --- /dev/null +++ b/paddle/operators/rnn_memory_helper_op.cc @@ -0,0 +1,153 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace operators { +class RNNMemoryHelperOp : public framework::OperatorBase { + public: + RNNMemoryHelperOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto mem_var_name = Input("X"); + auto *mem_var = scope.FindVar(mem_var_name); + PADDLE_ENFORCE(mem_var != nullptr, + "Cannot find mem_var in scope, mem_var_name is %s", + mem_var_name); + + auto out_name = this->Output("Out"); + auto *out_var = scope.FindVar(out_name); + PADDLE_ENFORCE(out_var != nullptr, + "Cannot find out_var in scope, out_var_name is %s", + out_name); + + auto *out_tensor = out_var->GetMutable(); + auto &mem_tensor = mem_var->Get(); + out_tensor->ShareDataWith(mem_tensor); + out_tensor->set_lod(mem_tensor.lod()); + } +}; + +class RNNMemoryHelperOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), ""); + PADDLE_ENFORCE(ctx->HasOutput("Out"), ""); + ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ "Out"); + } +}; + +class RNNMemoryHelperOpInfoMaker : public framework::OpProtoAndCheckerMaker { + public: + RNNMemoryHelperOpInfoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddOutput("Out", ""); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") + .SetDefault(framework::DataType::FP32); + AddComment(""); + } +}; + +class RNNMemoryHelperGradOp : public framework::OperatorBase { + public: + RNNMemoryHelperGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto out_grad_var_name = Input(framework::GradVarName("Out")); + auto *out_grad_var = scope.FindVar(out_grad_var_name); + + auto in_grad_var_name = Output(framework::GradVarName("X")); + auto *in_grad_var = scope.FindVar(in_grad_var_name); + PADDLE_ENFORCE(in_grad_var != nullptr, + "Cannot find in_grad_var in scope, name is %s", + in_grad_var_name); + + if (out_grad_var == nullptr) { + VLOG(5) << "Using fill constant 0 as starting gradient"; + auto in_var_name = Input("X"); + auto *in_var = scope.FindVar(in_var_name); + auto &in_var_tensor = in_var->Get(); + + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(in_var_tensor.type()); + attrs["shape"] = framework::vectorize2int(in_var_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {in_grad_var_name}}}, attrs); + zero_op->Run(scope, dev_ctx); + } else { + auto &out_grad_tensor = out_grad_var->Get(); + auto *in_grad_tensor = in_grad_var->GetMutable(); + in_grad_tensor->ShareDataWith(out_grad_tensor); + in_grad_tensor->set_lod(out_grad_tensor.lod()); + } + } +}; + +class RNNMemoryHelperGradOpInfoMaker + : public framework::OpProtoAndCheckerMaker { + public: + RNNMemoryHelperGradOpInfoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput(framework::GradVarName("Out"), ""); + AddInput("X", ""); + AddInput("Out", ""); + AddOutput(framework::GradVarName("X"), ""); + AddAttr("data_type", + "(int, default 5 (FP32)) " + "Output data type") + .SetDefault(framework::DataType::FP32); + AddComment(""); + } +}; + +class RNNMemoryHelperGradOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + auto x_grad_name = framework::GradVarName("X"); + PADDLE_ENFORCE(ctx->HasOutput(x_grad_name), ""); + PADDLE_ENFORCE(ctx->HasInput("X"), ""); + ctx->SetOutputDim(x_grad_name, ctx->GetInputDim("X")); + ctx->ShareLoD("X", /*->*/ x_grad_name); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(rnn_memory_helper, paddle::operators::RNNMemoryHelperOp, + paddle::operators::RNNMemoryHelperOpInfoMaker, + paddle::operators::RNNMemoryHelperOpShapeInference, + paddle::framework::DefaultGradOpDescMaker); +REGISTER_OPERATOR(rnn_memory_helper_grad, + paddle::operators::RNNMemoryHelperGradOp, + paddle::operators::RNNMemoryHelperGradOpInfoMaker, + paddle::operators::RNNMemoryHelperGradOpShapeInference); diff --git a/paddle/operators/save_op.cc b/paddle/operators/save_op.cc index 490256dfa1cf9b891713dac264e9260906ce1025..56909fb65f44ad00314103e21bee9535fbd59317 100644 --- a/paddle/operators/save_op.cc +++ b/paddle/operators/save_op.cc @@ -163,14 +163,19 @@ class SaveOpProtoMaker : public framework::OpProtoAndCheckerMaker { SaveOpProtoMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The tensor need to be saved"); - AddComment(R"DOC(Save operator -Save operator will serialize and write a tensor variable to disk file. + AddInput("X", "(Tensor ) Input tensor to be saved"); + AddComment(R"DOC( +Save operator + +This operator will serialize and write a tensor variable to file on disk. )DOC"); - AddAttr("overwrite", "Overwrite the output file if exist") + AddAttr("overwrite", + "(boolean, default true)" + "Overwrite the output file if exist") .SetDefault(true); AddAttr("file_path", - "Variable will be saved to \"file_path\".") + "(string)" + "The \"file_path\" where the variable will be saved.") .AddCustomChecker( [](const std::string &path) { return !path.empty(); }); } diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 5fcacf70d80527b4580a8f744ab3b79fb301d1d9..5745580504fb9bda551f21665bff5c65ae82aeb9 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -40,13 +40,16 @@ class ScaleOpMaker : public framework::OpProtoAndCheckerMaker { public: ScaleOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The input tensor of scale operator."); - AddOutput("Out", "The output tensor of scale operator."); - AddComment(R"DOC(Scale operator + AddInput("X", "(Tensor) Input tensor of scale operator."); + AddOutput("Out", "(Tensor) Output tensor of scale operator."); + AddComment(R"DOC( +Scale operator -The equation is: Out = scale*X +$$Out = scale*X$$ )DOC"); - AddAttr("scale", "The scaling factor of the scale operator.") + AddAttr("scale", + "(float, default 0)" + "The scaling factor of the scale operator.") .SetDefault(1.0); } }; diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc index 62e6c70b4513fdfab1c563b6b23f36292fb6486a..ce4b794bc35aca0912d89a4ae81a9aa0c73a2104 100644 --- a/paddle/operators/scatter_op.cc +++ b/paddle/operators/scatter_op.cc @@ -49,9 +49,11 @@ class ScatterOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("Ref")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Ref")->type()), + ctx.device_context()); } }; @@ -66,9 +68,11 @@ class ScatterGradOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("Ref")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Ref")->type()), + ctx.device_context()); } }; diff --git a/paddle/operators/seq_expand_op.cc b/paddle/operators/seq_expand_op.cc index 08fda9b44564249634f0d1a570e8b2458f88fd41..b862056ad400290a60e8a75a23dceeb1d4422ea4 100644 --- a/paddle/operators/seq_expand_op.cc +++ b/paddle/operators/seq_expand_op.cc @@ -53,8 +53,10 @@ class SeqExpandOpMaker : public framework::OpProtoAndCheckerMaker { "(LodTensor)The output of seq_expand op." "The lod of output will be as same as input(Y)'s lod."); AddComment(R"DOC( -Expand input(X) according to LOD of input(Y). +Seq Expand Operator. +This operator expands input(X) according to LOD of input(Y). +Following are cases to better explain how this works: Case 1: Given 2-level a LoDTensor input(X) diff --git a/paddle/operators/seq_expand_op.h b/paddle/operators/seq_expand_op.h index 8703105385183c1a0ee1a1b3831228f942c04dda..4ef0d02cf85c43e95335660be65a67df66b4f55c 100644 --- a/paddle/operators/seq_expand_op.h +++ b/paddle/operators/seq_expand_op.h @@ -32,7 +32,8 @@ class SeqExpandKernel : public framework::OpKernel { const T* x_data = x->data(); auto x_dims = x->dims(); auto* y = context.Input("Y"); - PADDLE_ENFORCE_EQ(x_dims[0], y->lod().back().size() - 1, + PADDLE_ENFORCE_EQ(static_cast(x_dims[0]), + y->lod().back().size() - 1, "The size of last lod level in Input(Y)" "must be equal to dims[0] of Input(X)."); out->set_lod(y->lod()); diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc index 46f73e3c279835bbb4bfdd7dede03a5535186b24..d1de0b444712a8c304c33bd194e306dfe3c41f02 100644 --- a/paddle/operators/sequence_concat_op.cc +++ b/paddle/operators/sequence_concat_op.cc @@ -47,19 +47,19 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", - "(A vector of LoDTensor), the input is a vector of LoDTensor, " + "(LodTensorArray) Input is a vector of LoDTensor, " "each of which is a variable-length sequence or nested sequence.") .AsDuplicable(); AddOutput("Out", - "(A LoDTensor), the variable-length output of " + "(LoDTensor), Variable-length output of " "sequence_concat Op."); AddAttr("axis", - "(int, default 0)" - "The axis which the inputs will be joined with. " + "(int, default 0) " + "The axis along which the inputs will be joined. " "If axis is 0, the inputs will be joined with LoD index.") .SetDefault(0); AddAttr("level", - "(int, default 0)" + "(int, default 0) " "The level at which the inputs will be joined. " "If the level is 0, the inputs will be joined at the nested " "sequence level. " @@ -68,34 +68,42 @@ class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker { "The level should be less than the level number of inputs.") .SetDefault(0); AddComment(R"DOC( - The sequence_concat operator concatenates multiple LoDTensors. - It only supports sequence (LoD Tensor with level number is 1) - or a nested sequence (LoD tensor with level number is 2) as its input. - - Case1: - If the axis is other than 0(here, axis is 1 and level is 1), - each input should have the same LoD information and the LoD - information of the output keeps the same as the input. - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) - LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) - - - Case2: - If the axis is 0(here, leve is 0), the inputs are concatenated along - time steps, the LoD information of the output need to re-compute. - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,2,3,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,1,2,3,4,5,6,7,9}}; Dims(Out) = (9,3,4) - - - Case3: - If the axis is 0(here, level is 1). - - LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) - LoD(x1) = {{0,3,5}, {0,1,3,4,5}}; Dims(x1) = (5,3,4) - LoD(Out) = {{0,5,9}, {0,2,5,7,9}}; Dims(Out) = (9,3,4) - - NOTE: The levels of all the inputs should be the same. +The sequence_concat operator concatenates multiple LoDTensors. +It only supports sequence (LoD Tensor with level number is 1) +or a nested sequence (LoD tensor with level number is 2) as its input. +- Case1: + If the axis is other than 0(here, axis is 1 and level is 1), + each input should have the same LoD information and the LoD + information of the output keeps the same as the input. + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4) + LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4) + +- Case2: + If the axis is 0(here, leve is 0), the inputs are concatenated along + time steps, the LoD information of the output need to re-compute. + The LoD information of level-1 should be same. + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,2,4}, {0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,2,4}, {0,2,5,8,11}}; Dims(Out) = (11,3,4) + +- Case3: + If the axis is 0(here, level is 1). + + LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,3,4}, {0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,5,8}, {0,1,2,3,5,7,8,9,11}}; Dims(Out) = (11,3,4) + +- Case4: + If the LoD number is 1, axis is 0, level is 0 + + LoD(x0) = {{0,1,2,3,4}}; Dims(x0) = (4,3,4) + LoD(x1) = {{0,1,3,5,7}}; Dims(x1) = (7,3,4) + LoD(Out) = {{0,2,5,8,11}}; Dims(Out) = (11,3,4) + +NOTE: The levels of all the inputs should be the same. )DOC"); } }; diff --git a/paddle/operators/sequence_concat_op.cu b/paddle/operators/sequence_concat_op.cu index 8dc4764785871262d21a5631cc9e8b805ba84244..9ca99c2258f547e6f9c23be0d394bc3ea2bb6678 100644 --- a/paddle/operators/sequence_concat_op.cu +++ b/paddle/operators/sequence_concat_op.cu @@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU - #include "paddle/operators/sequence_concat_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/sequence_concat_op.h b/paddle/operators/sequence_concat_op.h index 6adf96120c99f9b84a1ff947058e65ac3ddff1d4..09212070aa90b0f080f6140a312924229162aaec 100644 --- a/paddle/operators/sequence_concat_op.h +++ b/paddle/operators/sequence_concat_op.h @@ -24,28 +24,38 @@ using LoDTensor = framework::LoDTensor; using LoD = framework::LoD; template -LoD concatLoD(const std::vector ins, const size_t axis, - const size_t level) { +LoD ConcatLoD(const std::vector ins, const size_t level) { auto out_lod = ins[0]->lod(); + auto numLevels = ins[0]->NumLevels(); const size_t n = ins.size(); - if (axis == 0UL) { - for (size_t i = 1; i < n; ++i) { - for (size_t j = 0; j < ins[i]->lod()[0].size(); ++j) { - out_lod[0][j] += ins[i]->lod()[0][j]; - } + const size_t level_idx = ins[0]->NumLevels() - 1 - level; + for (size_t i = 1; i < n; ++i) { + for (size_t j = 0; j < ins[i]->lod()[level_idx].size(); ++j) { + out_lod[level_idx][j] += ins[i]->lod()[level_idx][j]; + } + } - if (ins[0]->NumLevels() == 2) { - for (size_t j = 1; j < ins[i]->lod()[1].size(); ++j) { - if (level == 0UL) { - out_lod[1].push_back(out_lod[1].back() + ins[i]->lod()[1][j] - - ins[i]->lod()[1][j - 1]); - } else if (level == 1UL) { - out_lod[1][j] += ins[1]->lod()[1][j]; - } + for (size_t i = level_idx; i < numLevels - 1; ++i) { + size_t lod_len = 1; + for (size_t j = 0; j < n; ++j) { + lod_len += ins[j]->lod()[i + 1].size() - 1; + } + out_lod[i + 1].clear(); + out_lod[i + 1].resize(lod_len); + + size_t idx = 1; + for (size_t j = 0; j < ins[0]->lod()[i].size() - 1; ++j) { + for (size_t k = 0; k < n; ++k) { + for (size_t m = ins[k]->lod()[i][j]; m < ins[k]->lod()[i][j + 1]; ++m) { + out_lod[i + 1][idx] = out_lod[i + 1][idx - 1] + + ins[k]->lod()[i + 1][m + 1] - + ins[k]->lod()[i + 1][m]; + idx++; } } } } + return out_lod; } @@ -82,18 +92,21 @@ class SequenceConcatOpKernel : public framework::OpKernel { "should be greater than the specify level"); out->mutable_data(ctx.GetPlace()); - auto out_lod = concatLoD(ins, axis, level); + auto out_lod = ins[0]->lod(); + if (axis == 0) { + out_lod = ConcatLoD(ins, level); + } out->set_lod(out_lod); - auto out_lod_level = out_lod[level]; + const size_t level_idx = out_lod.size() - level - 1; + auto out_lod_level = framework::ToAbsOffset(out_lod)[level_idx]; for (size_t i = 0; i < out_lod_level.size() - 1; ++i) { Tensor out_t = out->Slice(static_cast(out_lod_level[i]), static_cast(out_lod_level[i + 1])); auto out_stride = framework::stride(out_t.dims()); size_t offset = 0; - for (size_t j = 0; j < n; ++j) { - auto in_lod_level = ins[j]->lod()[level]; + auto in_lod_level = framework::ToAbsOffset(ins[j]->lod())[level_idx]; auto in_stride = framework::stride(ins[j]->dims()); Tensor in_t = ins[j]->Slice(static_cast(in_lod_level[i]), static_cast(in_lod_level[i + 1])); @@ -124,9 +137,12 @@ class SequenceConcatGradOpKernel : public framework::OpKernel { x_grads[i]->set_lod(ins[i]->lod()); x_grads[i]->mutable_data(ctx.GetPlace()); } - - auto out_lod = concatLoD(ins, axis, level); - auto out_lod_level = out_lod[level]; + auto out_lod = ins[0]->lod(); + if (axis == 0UL) { + out_lod = ConcatLoD(ins, level); + } + const size_t level_idx = out_lod.size() - level - 1; + auto out_lod_level = framework::ToAbsOffset(out_lod)[level_idx]; for (size_t i = 0; i < out_lod_level.size() - 1; ++i) { Tensor out_grad_t = @@ -136,7 +152,8 @@ class SequenceConcatGradOpKernel : public framework::OpKernel { size_t offset = 0; for (size_t j = 0; j < n; ++j) { - auto x_grad_lod_level = x_grads[j]->lod()[level]; + auto x_grad_lod_level = + framework::ToAbsOffset(x_grads[j]->lod())[level_idx]; auto x_grad_stride = framework::stride(x_grads[j]->dims()); Tensor x_grad_t = x_grads[j]->Slice(static_cast(x_grad_lod_level[i]), diff --git a/paddle/operators/sequence_conv_op.cc b/paddle/operators/sequence_conv_op.cc index a3f2ed14439572e9723c3057d212bb773b2a4e44..41cadce4c603a9c14db79e2f6b30f8664cf72a38 100644 --- a/paddle/operators/sequence_conv_op.cc +++ b/paddle/operators/sequence_conv_op.cc @@ -105,10 +105,10 @@ class SequenceConvOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", - "(LoDTensor) the input(X) is a LodTensor, which support " + "(LoDTensor) the input(X) is a LodTensor, which supports " "variable-time length input sequence. The underlying tensor in " - "this LoDTensor is a matrix with shape (T, N), where, T is the " - "total time steps in this mini-batch, N is the input_hidden_size."); + "this LoDTensor is a matrix with shape (T, N), where T is the " + "total time steps in this mini-batch and N is the input_hidden_size."); AddInput("PaddingData", "(Tensor, optional) the input(PaddingData) is an optional " "parameter, and it is learnable. " @@ -157,14 +157,16 @@ class SequenceConvOpMaker : public framework::OpProtoAndCheckerMaker { .GreaterThan(0); AddComment(R"DOC( - SequenceConvOp performs convolution operation on features of - contextLength time-steps of each instance. - The convolution operation calculates the output based on the input, filter - and strides, paddings parameters. The size of each dimension of the - parameters is checked in the infer-shape. In order to ensure the equal - length of sequence before and after convolution, it is necessary to fill - the top and bottom of each sequence according to context_length, - context_stride and context_start. +Sequence Conv Operator. + +SequenceConvOp performs convolution operation on features of contextLength +time-steps of each instance. The convolution operation calculates the output +based on the input, filter, strides and paddings parameters. +The size of each dimension of the parameters is checked during infer-shape. +In order to ensure the equal length of sequence before and after convolution, +it is necessary to fill the top and bottom of each sequence based on +context_length, context_stride and context_start. + )DOC"); } }; diff --git a/paddle/operators/sequence_pool_op.cc b/paddle/operators/sequence_pool_op.cc index 29d19df10898634dd433abc1263fefe169de6f08..2a000ac60b176737277605c3ac812ea65a0e03fc 100644 --- a/paddle/operators/sequence_pool_op.cc +++ b/paddle/operators/sequence_pool_op.cc @@ -27,6 +27,11 @@ class SequencePoolOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of SequencePoolOp should not be null."); ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + if (ctx->Attrs().Get("pooltype") == "MAX") { + PADDLE_ENFORCE(ctx->HasOutput("MaxIndex"), + "Output(MaxIndex) of SequencePoolOp should not be null."); + ctx->SetOutputDim("MaxIndex", ctx->GetInputDim("X")); + } } }; @@ -35,42 +40,50 @@ class SequencePoolOpMaker : public framework::OpProtoAndCheckerMaker { SequencePoolOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "(LoDTensor), the variable-length input of SequencePoolOp"); + AddInput("X", "(LoDTensor) The variable-length input of SequencePoolOp"); AddOutput("Out", - "(Tensor), output of SequencePoolOp, which does not contain LoD " + "(Tensor) The output of SequencePoolOp does not contain LoD " "infomation."); + AddOutput("MaxIndex", + "(Tensor) This tensor is used for the sequence max-pooling " + "to record the max indexes.") + .AsIntermediate(); AddAttr( "pooltype", "(int, default AVERAGE) the pooling pooltype of SequencePoolOp.") - .SetDefault("AVERAGE"); + .SetDefault("AVERAGE") + .InEnum({"AVERAGE", "SUM", "SQRT", "LAST", "FIRST", "MAX"}); AddComment(R"DOC( - SequencePoolOp pools features of all time-steps of each instance. - - It supports six pooling pooltype: - - AVERAGE: Out[i] = average_{for each instance in i-th sequence}{X[i]} - - SUM: Out[i] = sum_{for each instance in i-th sequence}{X[i]} - - SQRT: Out[i] = sum_{for each instance in i-th sequence}{X[i]} - / sqrt(i-th sequence length) - - LAST: Out[i] = last instance in i-th sequence X[i] - - FIRST: Out[i] = first instance in i-th sequence X[i] - - MAX: Out[i] = max_{for each instance in i-th sequence}{X[i]} - - For a mini-batch of 3 variable-length sentences, containing 2, 3, and 2 time-steps: - - Assume X is a [7,M,N] LoDTensor, and X->lod()[0] = [0, 2, 5, 7], 7=2+3+2. - Besides, for the sake of simplicity, we assume M=1 and N=1, - and the value of X = [[1, 3], [2, 4, 6], [5, 1]]. - - Thus, Out is a [3,1,1] Tensor without LoD infomation. - And for different pooltype, the value of Out is as follows: - - - AVERAGE: [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 - - SUM: [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 - - SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), +Sequence Pool Operator. + +The SequencePoolOp pools features of all time-steps of each instance. +It supports six pooling types: +1. AVERAGE: Out[i] = $$avg(X_i)$$ +2. SUM: Out[i] = $$\sum_jX_{ij}$$ +3. SQRT: Out[i] = $$\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}$$ +4. LAST: Out[i] = last instance in i-th sequence X[i] +5. FIRST: Out[i] = first instance in i-th sequence X[i] +6. MAX: Out[i] = $$max(X_i)$$ + +The following example explains how this works: +For a mini-batch of 3 variable-length sentences, +containing 2, 3, and 2 time-steps: + +Assume X is a [7,M,N] LoDTensor, and X->lod()[0] = [0, 2, 5, 7], 7=2+3+2. +Besides, for the sake of simplicity, we assume M=1 and N=1, +and the value of X = [[1, 3], [2, 4, 6], [5, 1]]. + +Thus, Out is a [3,1,1] Tensor without LoD infomation. +And for different pooltype, the value of Out is as follows: + +- AVERAGE: [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 +- SUM: [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 +- SQRT: [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), 6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2) - - MAX: [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) - - LAST: [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) - - FIRST: [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) +- MAX: [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) +- LAST: [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) +- FIRST: [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) + )DOC"); } }; @@ -92,6 +105,14 @@ class SequencePoolGradOp : public framework::OperatorWithKernel { } ctx->SetOutputDim(framework::GradVarName("X"), x_dims); } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + ctx.device_context()); + } }; } // namespace operators diff --git a/paddle/operators/sequence_pool_op.h b/paddle/operators/sequence_pool_op.h index e0e0493fe0ef7e1963ce5c2e3f37c164a605809b..7f136d8cf0e1eaae7b4de32988b60ae8a5034cc6 100644 --- a/paddle/operators/sequence_pool_op.h +++ b/paddle/operators/sequence_pool_op.h @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/sequence_pooling.h" namespace paddle { namespace operators { @@ -34,7 +35,7 @@ class SequencePoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* in = context.Input("X"); - auto* out = context.Output("Out"); + auto* out = context.Output("Out"); std::string pooltype = context.Attr("pooltype"); auto dims = in->dims(); @@ -53,6 +54,16 @@ class SequencePoolKernel : public framework::OpKernel { auto lod_level_0 = lod[0]; out->mutable_data(context.GetPlace()); + + if (pooltype == "MAX") { + math::MaxSeqPoolFunctor max_pool; + auto* index = context.Output("MaxIndex"); + index->Resize({dims}); + index->mutable_data(context.GetPlace()); + max_pool(context.device_context(), *in, out, index); + return; + } + auto place = context.GetEigenDevice(); for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { Tensor in_t = in->Slice(static_cast(lod_level_0[i]), @@ -69,8 +80,6 @@ class SequencePoolKernel : public framework::OpKernel { } else if (pooltype == "SQRT") { out_e.device(place) = in_e.sum(Eigen::array({{0}})) / std::sqrt(static_cast(h)); - } else if (pooltype == "MAX") { - out_e.device(place) = in_e.maximum(Eigen::array({{0}})); } else if (pooltype == "LAST") { out_e.device(place) = in_e.chip(h - 1, 0); } else if (pooltype == "FIRST") { @@ -87,8 +96,8 @@ class SequencePoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* in = context.Input("X"); + auto* out_g = context.Input(framework::GradVarName("Out")); auto* in_g = context.Output(framework::GradVarName("X")); - auto* out_g = context.Input(framework::GradVarName("Out")); std::string pooltype = context.Attr("pooltype"); auto dims = in->dims(); @@ -96,6 +105,14 @@ class SequencePoolGradKernel : public framework::OpKernel { int64_t w = in->numel() / dims[0]; in_g->mutable_data(context.GetPlace()); + + if (pooltype == "MAX") { + math::MaxSeqPoolGradFunctor max_pool_grad; + auto* index = context.Input("MaxIndex"); + max_pool_grad(context.device_context(), *out_g, *index, in_g); + return; + } + if (pooltype == "LAST" || pooltype == "FIRST") { // set X@Grad be zero at first when pooltype is LAST/FIRST math::SetConstant functor; @@ -109,6 +126,7 @@ class SequencePoolGradKernel : public framework::OpKernel { int64_t h = static_cast(lod[i + 1] - lod[i]); auto in_g_e = EigenMatrix::From(in_g_t, {h, w}); auto out_g_e = EigenMatrix::From(out_g_t, {1, w}); + auto out_g_e_v = EigenVector::Flatten(out_g_t); Eigen::DSizes bcast(h, 1); if (pooltype == "AVERAGE") { @@ -118,24 +136,10 @@ class SequencePoolGradKernel : public framework::OpKernel { } else if (pooltype == "SQRT") { in_g_e.device(place) = (out_g_e / std::sqrt(static_cast(h))).broadcast(bcast); - } else if (pooltype == "MAX") { - auto in_t = - in->Slice(static_cast(lod[i]), static_cast(lod[i + 1])); - Eigen::Map> - in_t_map(in_t.data(), h, w); - int row_id; - Eigen::array extents{{1, 1}}; - for (int col_id = 0; col_id < w; col_id++) { - in_t_map.col(col_id).maxCoeff(&row_id); - Eigen::array in_offsets{{row_id, col_id}}; - Eigen::array out_offsets{{0, col_id}}; - in_g_e.slice(in_offsets, extents).device(place) = - out_g_e.slice(out_offsets, extents); - } } else if (pooltype == "LAST") { - in_g_e.chip(h - 1, 0).device(place) = out_g_e; + in_g_e.chip(h - 1, 0).device(place) = out_g_e_v; } else if (pooltype == "FIRST") { - in_g_e.chip(0, 0).device(place) = out_g_e; + in_g_e.chip(0, 0).device(place) = out_g_e_v; } else { PADDLE_THROW("unsupported pooling pooltype"); } diff --git a/paddle/operators/sequence_softmax_op.cc b/paddle/operators/sequence_softmax_op.cc index c891ab1fdcbb167453462c45b00b4632e663dd0e..32c15025660ebf0baf317e269a33c047e6844219 100644 --- a/paddle/operators/sequence_softmax_op.cc +++ b/paddle/operators/sequence_softmax_op.cc @@ -43,20 +43,24 @@ class SequenceSoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { "(LoDTensor) 1-D or 2-D output LoDTensor with the 2-nd dimension " "of length 1."); AddComment(R"DOC( -SequenceSoftmaxOp computes softmax activation among all time-steps for each +Sequence Softmax Operator. + +SequenceSoftmaxOp computes the softmax activation among all time-steps for each sequence. The dimension of each time-step should be 1. Thus, the shape of -input Tensor can be either [N, 1] or [N], where N is the sum of all sequences' -lengths. +input Tensor can be either [N, 1] or [N], where N is the sum of the length +of all sequences. -Equation: +The algorithm works as follows: for i-th sequence in a mini-batch: - Out(X[lod[i]:lod[i+1]], :) = - exp(X[lod[i]:lod[i+1], :]) / sum(exp(X[lod[i]:lod[i+1], :])) + $$Out(X[lod[i]:lod[i+1]], :) = + \frac{\exp(X[lod[i]:lod[i+1], :])} + {\sum(\exp(X[lod[i]:lod[i+1], :]))}$$ For example, for a mini-batch of 3 sequences with variable-length, each containing 2, 3, 2 time-steps, the lod of which is [0, 2, 5, 7], then softmax will be computed among X[0:2, :], X[2:5, :], X[5:7, :] and N turns out to be 7. + )DOC"); } }; diff --git a/paddle/operators/sequence_softmax_op.cu b/paddle/operators/sequence_softmax_op.cu index f2a1e3d5e31ef21b95a51b287bdd1d4aa9221e89..7023795a3b5777c250a9323a304a54849d763e9e 100644 --- a/paddle/operators/sequence_softmax_op.cu +++ b/paddle/operators/sequence_softmax_op.cu @@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU - #include "paddle/operators/sequence_softmax_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/sequence_softmax_op.h b/paddle/operators/sequence_softmax_op.h index 3eb1e2844dff6ac94e86dcf4586bb51bc33adbec..1b68dd0662ddfffc57b187945fe131e202c55174 100644 --- a/paddle/operators/sequence_softmax_op.h +++ b/paddle/operators/sequence_softmax_op.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/softmax.h" diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 939176c73dc21dc662b1aaf23d8077c6856a5650..72f4e4d5cbcd692423fa2a3e9ec8e7033b552c3c 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -45,15 +45,17 @@ class SGDOpMaker : public framework::OpProtoAndCheckerMaker { public: SGDOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Param", "Input parameter"); - AddInput("LearningRate", "Learning rate of SGD"); - AddInput("Grad", "Input gradient"); - AddOutput("ParamOut", "output parameter"); + AddInput("Param", "(Tensor) Input parameter"); + AddInput("LearningRate", "(Tensor) Learning rate of SGD"); + AddInput("Grad", "(Tensor) Input gradient"); + AddOutput("ParamOut", "(Tensor) Output parameter"); AddComment(R"DOC( -Simplest sgd algorithm. +SGD operator -param_out = param - learning_rate * grad; +This operator implements one step of the stochastic gradient descent algorithm. + +$$param_out = param - learning_rate * grad$$ )DOC"); } diff --git a/paddle/operators/shrink_rnn_memory_op.cc b/paddle/operators/shrink_rnn_memory_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..65bccc0c81d0ad9674649933a20ec7b09fec5b37 --- /dev/null +++ b/paddle/operators/shrink_rnn_memory_op.cc @@ -0,0 +1,149 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/framework/lod_rank_table.h" +#include "paddle/operators/array_operator.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +class ShrinkRNNMemoryOp : public ArrayOp { + public: + ShrinkRNNMemoryOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x_var = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x_var != nullptr, "Input X must be set"); + auto &x_tensor = x_var->Get(); + size_t offset = this->GetOffset(scope, dev_ctx); + auto *rank_table_var = scope.FindVar(Input("RankTable")); + PADDLE_ENFORCE(rank_table_var != nullptr, "RankTable must be set"); + auto &rank_table = rank_table_var->Get(); + + auto &rank_items = rank_table.items(); + int dst_num_rows = + std::lower_bound(rank_items.begin(), rank_items.end(), offset, + [](const framework::LoDRankTable::TableItem &a, + size_t b) { return a.length > b; }) - + rank_items.begin(); + + auto *out_var = scope.FindVar(Output("Out")); + PADDLE_ENFORCE(out_var != nullptr, "Output Out must be set"); + auto &out_tensor = *out_var->GetMutable(); + if (dst_num_rows != 0) { + out_tensor.ShareDataWith(x_tensor.Slice(0, dst_num_rows)); + } + } +}; + +class ShrinkRNNMemoryOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ShrinkRNNMemoryOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddInput("RankTable", ""); + AddInput("I", ""); + AddOutput("Out", ""); + AddComment(""); + } +}; + +class ShrinkRNNMemoryInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasInput("I")); + PADDLE_ENFORCE(context->HasInput("RankTable")); + context->SetOutputDim("Out", context->GetInputDim("X")); + } +}; + +class ShrinkRNNMemoryGradOp : public ArrayOp { + public: + ShrinkRNNMemoryGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *dout_var = scope.FindVar(Input(framework::GradVarName("Out"))); + auto *dx_var = scope.FindVar(Output(framework::GradVarName("X"))); + PADDLE_ENFORCE(dx_var != nullptr, "Input Gradient should not be nullptr"); + auto *x_var = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x_var != nullptr); + + auto &x_tensor = x_var->Get(); + auto &dx_tensor = *dx_var->GetMutable(); + dx_tensor.Resize(x_tensor.dims()); + dx_tensor.mutable_data(x_tensor.place(), x_tensor.type()); + + if (dout_var == nullptr) { // dx_tensor fill zero + math::set_constant(dev_ctx, &dx_tensor, 0.0f); + } else { + auto &dout_tensor = dout_var->Get(); + auto height = dout_tensor.dims()[0]; + dx_tensor.Slice(0, static_cast(height)) + .CopyFrom(dout_tensor, dout_tensor.place(), dev_ctx); + if (dx_tensor.dims()[0] < height) { + auto rest_tensor = dx_tensor.Slice( + static_cast(height), static_cast(dout_tensor.dims()[0])); + math::set_constant(dev_ctx, &rest_tensor, 0.0f); + } + } + } +}; + +class ShrinkRNNMemoryGradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X")); + PADDLE_ENFORCE(context->HasOutput(framework::GradVarName("X"))); + context->SetOutputDim(framework::GradVarName("X"), + context->GetInputDim("X")); + } +}; + +class ShrinkRNNGradOpMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *op = new framework::OpDescBind(); + op->SetType("shrink_rnn_memory_grad"); + op->SetInput("X", Input("X")); + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + op->SetAttrMap(Attrs()); + return std::unique_ptr(op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(shrink_rnn_memory, ops::ShrinkRNNMemoryOp, + ops::ShrinkRNNMemoryInferShape, + ops::ShrinkRNNMemoryOpProtoMaker, ops::ShrinkRNNGradOpMaker); +REGISTER_OPERATOR(shrink_rnn_memory_grad, ops::ShrinkRNNMemoryGradOp, + ops::ShrinkRNNMemoryGradInferShape); diff --git a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc index e781c8db208464cb94d94d1914e50f5aba3db2c6..d9e40546523c60b0a7eec2e0593446258996ba58 100644 --- a/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc +++ b/paddle/operators/sigmoid_cross_entropy_with_logits_op.cc @@ -107,26 +107,28 @@ class SigmoidCrossEntropyWithLogitsOpMaker AddComment(R"DOC( SigmoidCrossEntropyWithLogits Operator. -This measures the elementwise probability error in discrete classification tasks +This measures the element-wise probability error in classification tasks in which each class is independent. This can be thought of as predicting labels -for a data-point that are not mutually exclusive. For example, a news article -can be about politics, technology or sports at the same time or none of these. +for a data-point, where labels are not mutually exclusive. +For example, a news article can be about politics, technology or sports +at the same time or none of these. The logistic loss is given as follows: - loss = -Labels * log(sigmoid(X)) - (1 - Labels) * log(1 - sigmoid(X)) + $$loss = -Labels * \log(\sigma(X)) - (1 - Labels) * \log(1 - \sigma(X))$$ -We know that sigmoid(X) = (1 / (1 + exp(-X))). By substituting this we get +We know that $$\sigma(X) = (1 / (1 + \exp(-X)))$$. By substituting this we get: - loss = X - X * Labels + log(1 + exp(-X)) + $$loss = X - X * Labels + \log(1 + \exp(-X))$$ -For stability and to prevent overflow of exp(-X) when X < 0, -we can reformulate the loss as follows: +For stability and to prevent overflow of $$\exp(-X)$$ when X < 0, +we reformulate the loss as follows: - loss = max(X, 0) - X * Labels + log(1 + exp(-abs(X))) + $$loss = \max(X, 0) - X * Labels + \log(1 + \exp(-|X|))$$ Both the input `X` and `Labels` can carry the LoD (Level of Details) information. However the output only shares the LoD with input `X`. + )DOC"); } }; diff --git a/paddle/operators/sign_op.cc b/paddle/operators/sign_op.cc index 1b2f879d6d305e4e77be41683d8249904337a6f8..08bf2e4e7cc101a3bcc907d3b40ee82347b39f80 100644 --- a/paddle/operators/sign_op.cc +++ b/paddle/operators/sign_op.cc @@ -38,9 +38,10 @@ class SignOpMaker : public framework::OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) Input tensor of sign operator."); AddOutput("Out", "(Tensor) Output tensor of sign operator."); - AddComment(R"DOC(Sign operator + AddComment(R"DOC( +Sign operator -The equation is: Out = X.sign() +$$Out = X.sign()$$ )DOC"); } }; diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index 758481943d463f22eb6c6e0be9a99ad99161da5b..ebf7b43700a7498aa18b5f648b0b8c2c4e7b442b 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -77,14 +77,17 @@ class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { "A float scalar with default value 3.0.") .SetDefault(3.0); AddComment(R"DOC( -Compute smooth l1 loss for input and target. The operator take the 1st -dimension of input as batch size. For each instance, it will compute -smooth l1 loss element by element first and sum all losses to one value. -So the output shape is [batch_size, 1]. +Smooth L1 Loss Operator. + +This operator computes the smooth l1 loss for input and target. +The operator takes the first dimension of input as the batch size. +For each instance, it computes the smooth l1 loss element by element first +and then sums all the losses. So the resulting output shape +is [batch_size, 1]. The equation is: -loss = 0.5 * (sigma * (x-y))^2 if abs(x - y) < 1 / sigma^2 - abs(x - y) - 0.5 / sigma^2 otherwise +loss = $$0.5 * (\sigma * (x-y))^2$$ if $$|x - y| < 1 /({\sigma}^2)$$ + $$\frac{|x - y| - 0.5}{{\sigma}^2}$$ otherwise )DOC"); } diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index 00fd0b32a9b3c0dd9fedf7b7621b1f15e5c4ce93..93f89e33a73c5f4c6c0e5a8793a0abe7c692b656 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -44,20 +44,23 @@ class SoftmaxOpMaker : public framework::OpProtoAndCheckerMaker { "2-D with shape [batch_size, input_feature_dimensions]."); AddOutput("Y", "The normalized values with the same shape as X."); AddComment(R"DOC( -The input of softmax operator is a 2-D tensor with shape N x K (N is the +Softmax Operator. + +The input of the softmax operator is a 2-D tensor with shape N x K (N is the batch_size, K is the dimension of input feature). The output tensor has the same shape as the input tensor. For each row of the input tensor, the softmax operator squashes the K-dimensional vector of arbitrary real values to a K-dimensional vector of real -values in the range [0, 1] that add up to 1. Specifically, it computes the -exponential of the given dimension and the sum of exponential values of all -the other dimensions in the K-dimensional vector input. Then the ratio of the -exponential of the given dimension and the sum of exponential values of all -the other dimensions is the output of the softmax operator. +values in the range [0, 1] that add up to 1. +It computes the exponential of the given dimension and the sum of exponential +values of all the other dimensions in the K-dimensional vector input. +Then the ratio of the exponential of the given dimension and the sum of +exponential values of all the other dimensions is the output of the softmax +operator. For each row `i` and each column `j` in input X, we have: - Y[i, j] = exp(X[i, j]) / sum_j(exp(X[i, j])) + $$Y[i, j] = \frac{\exp(X[i, j])}{\sum_j(exp(X[i, j])}$$ )DOC"); } diff --git a/paddle/operators/softmax_op.cu b/paddle/operators/softmax_op.cu index 2e99a89699dbdcafc8055c47debf9e49f10507e6..013ace19ae3d4a1af29b570ba33fea3e4595fe5b 100644 --- a/paddle/operators/softmax_op.cu +++ b/paddle/operators/softmax_op.cu @@ -12,7 +12,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -#define EIGEN_USE_GPU #include "paddle/operators/softmax_op.h" namespace ops = paddle::operators; diff --git a/paddle/operators/softmax_op.h b/paddle/operators/softmax_op.h index 2c08853f4f615bfe95f51aa20776ddddcdaa8f61..44d1e63f1bb4798144218cd1caf01f133825bcff 100644 --- a/paddle/operators/softmax_op.h +++ b/paddle/operators/softmax_op.h @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/softmax.h" @@ -21,9 +20,6 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; template class SoftmaxKernel : public framework::OpKernel { diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index 50497da1b70d39d2638240dd91035c9181124af9..ed96e8cee5a78e63ea29ed383d06c1258abdc328 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -4,13 +4,13 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #include "paddle/operators/softmax_with_cross_entropy_op.h" #include @@ -30,12 +30,10 @@ class SoftmaxWithCrossEntropyOpMaker "which is a 2-D tensor with shape [N x K]. N is the batch_size, " "and K is the class number."); AddInput("Label", - "(Tensor, default: Tensor), The ground truth which is a 2-D " - "tensor. " - "If softLabel is set to false, Label is a Tensor with shape " - "[N x 1]." - "If softLabel is set to true, Label is a Tensor " - "with shape [N x K]."); + "(Tensor) The ground truth which is a 2-D tensor. If soft_label " + "is set to false, Label is a Tensor with shape [N x 1]. If " + "soft_label is set to true, Label is a Tensor with " + "shape [N x K]."); AddOutput( "Softmax", "(Tensor, default: Tensor), A 2-D tensor with shape [N x K]. " @@ -51,32 +49,34 @@ class SoftmaxWithCrossEntropyOpMaker "the given labels as soft labels.") .SetDefault(false); AddComment(R"DOC( -Cross entropy loss with softmax are used as the output layer extensively. This +Softmax With Cross Entropy Operator. + +Cross entropy loss with softmax is used as the output layer extensively. This operator computes the softmax normalized values for each row of the input -tensor, after which cross-entropy loss is then computed. This provides a more +tensor, after which cross-entropy loss is computed. This provides a more numerically stable gradient. -Because this operators performs a softmax on logits internally, it expects -unscaled logits. Please do not call this op with the output of softmax operator, -which will produce incorrect results. +Because this operator performs a softmax on logits internally, it expects +unscaled logits. This operator should not be used with the output of +softmax operator since that would produce incorrect results. -When the attribute softLabel is set false, this operators expects mutually -exclusive hard labels, each sample in a batch is in exactly one class with -probabilities 1. Each sample in the batch with one and only one label. +When the attribute soft_label is set false, this operators expects mutually +exclusive hard labels, each sample in a batch is in exactly one class with a +probability of 1.0. Each sample in the batch will have a single label. -Equation: +The equation is as follows: -1) hard label (one-hot label) +1) Hard label (one-hot label, so every sample has exactly one class) -Loss_j = \f$ -\text{Logit}_{Label_j} + +$$Loss_j = \f$ -\text{Logit}_{Label_j} + \log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right), -j = 1, ..., K $\f +j = 1, ..., K $\f$$ -2) soft label (a distribution over all classes) +2) Soft label (each sample can have a distribution over all classes) -Loss_j = \f$ -\sum_{i=0}^{K}\text{Label}_i\left(\text{Logit}_i - +$$Loss_j = \f$ -\sum_{i=0}^{K}\text{Label}_i\left(\text{Logit}_i - \log\left(\sum_{i=0}^{K}\exp(\text{Logit}_i)\right)\right), -j = 1,...,K $\f +j = 1,...,K $\f$$ )DOC"); } @@ -121,9 +121,11 @@ class SoftmaxWithCrossEntropyOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType(ctx.Input("Logits")->type()); + return framework::OpKernelType( + framework::ToDataType(ctx.Input("Logits")->type()), + ctx.device_context()); } }; @@ -160,10 +162,12 @@ class SoftmaxWithCrossEntropyOpGrad : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return framework::ToDataType( - ctx.Input(framework::GradVarName("Loss"))->type()); + return framework::OpKernelType( + framework::ToDataType( + ctx.Input(framework::GradVarName("Loss"))->type()), + ctx.device_context()); } }; @@ -196,6 +200,8 @@ REGISTER_OPERATOR(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp, REGISTER_OPERATOR(softmax_with_cross_entropy_grad, ops::SoftmaxWithCrossEntropyOpGrad); REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy, - ops::SoftmaxWithCrossEntropyKernel); + ops::SoftmaxWithCrossEntropyKernel, + ops::SoftmaxWithCrossEntropyKernel); REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy_grad, - ops::SoftmaxWithCrossEntropyGradKernel); + ops::SoftmaxWithCrossEntropyGradKernel, + ops::SoftmaxWithCrossEntropyGradKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.cu b/paddle/operators/softmax_with_cross_entropy_op.cu index 7602918bb39312db3c4d1a4064801712ef94ec72..b1faddac3fd21aaf817caf9d3e57e664f4e0e2d5 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cu +++ b/paddle/operators/softmax_with_cross_entropy_op.cu @@ -4,13 +4,13 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #define EIGEN_USE_GPU @@ -24,7 +24,7 @@ using Tensor = framework::Tensor; namespace { template __global__ void CrossEntropyGrad(T* logit_grad, const T* loss_grad, - const int* labels, const int batch_size, + const int64_t* labels, const int batch_size, const int class_num) { int tid = blockIdx.x * blockDim.x + threadIdx.x; int sample_idx = tid / class_num; @@ -50,7 +50,7 @@ __global__ void SoftCrossEntropyGradientKernel(T* logit_grad, int ids = blockIdx.x * blockDim.x + threadIdx.x; if (ids < batch_size * class_num) { int row_ids = ids / class_num; - logit_grad[ids] = logit_grad[ids] * (loss_grad[row_ids] - labels[ids]); + logit_grad[ids] = loss_grad[row_ids] * (logit_grad[ids] - labels[ids]); } } } // namespace @@ -104,7 +104,7 @@ class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { .stream()>>>(logit_grad_data, loss_grad_data, label_data, batch_size, class_num); } else { - const int* label_data = labels->data(); + const int64_t* label_data = labels->data(); CrossEntropyGrad<<< grid, block, 0, reinterpret_cast( context.device_context()) @@ -119,6 +119,8 @@ class SoftmaxWithCrossEntropyGradCUDAKernel : public framework::OpKernel { namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy, - ops::SoftmaxWithCrossEntropyCUDAKernel); + ops::SoftmaxWithCrossEntropyCUDAKernel, + ops::SoftmaxWithCrossEntropyCUDAKernel); REGISTER_OP_GPU_KERNEL(softmax_with_cross_entropy_grad, - ops::SoftmaxWithCrossEntropyGradCUDAKernel); + ops::SoftmaxWithCrossEntropyGradCUDAKernel, + ops::SoftmaxWithCrossEntropyGradCUDAKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.h b/paddle/operators/softmax_with_cross_entropy_op.h index 7f3f9e23aa9455437cfa893363b3e59a0699dbea..c4ab3f74b4b07d13957d99e01aa4868fac719f61 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.h +++ b/paddle/operators/softmax_with_cross_entropy_op.h @@ -4,13 +4,13 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #pragma once #include "paddle/framework/eigen.h" @@ -60,25 +60,25 @@ class SoftmaxWithCrossEntropyGradKernel : public framework::OpKernel { logit_grad->ShareDataWith(*context.Input("Softmax")); const int class_num = logit_grad->dims()[1]; + auto out_grad_mat = EigenMatrix::From(*out_grad); + auto logit_grad_mat = EigenMatrix::From(*logit_grad); + if (context.Attr("soft_label")) { - auto out_grad_mat = EigenMatrix::From(*out_grad); - auto logit_grad_mat = EigenMatrix::From(*logit_grad); auto lbl_mat = EigenMatrix::From(*labels); - logit_grad_mat.device(context.GetEigenDevice()) = - logit_grad_mat * - (out_grad_mat.broadcast(Eigen::DSizes(1, class_num)) - - lbl_mat); + out_grad_mat.broadcast(Eigen::DSizes(1, class_num)) * + (logit_grad_mat - lbl_mat); } else { + logit_grad_mat.device(context.GetEigenDevice()) = + logit_grad_mat * + out_grad_mat.broadcast(Eigen::DSizes(1, class_num)); + const int batch_size = logit_grad->dims()[0]; - const int* label_data = labels->data(); - const T* out_grad_data = out_grad->data(); + const int64_t* label_data = labels->data(); T* logit_grad_data = logit_grad->data(); - + const T* out_grad_data = out_grad->data(); for (int i = 0; i < batch_size; ++i) { - int index = i * class_num + label_data[i]; - logit_grad_data[index] = - out_grad_data[i] * (logit_grad_data[index] - 1.); + logit_grad_data[i * class_num + label_data[i]] -= out_grad_data[i]; } } } diff --git a/paddle/operators/split_lod_tensor_op.cc b/paddle/operators/split_lod_tensor_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..db635f2ba0804143c9a2e04ff006dfbc8744f3fc --- /dev/null +++ b/paddle/operators/split_lod_tensor_op.cc @@ -0,0 +1,186 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/memory/memcpy.h" + +namespace paddle { +namespace operators { + +struct CopyRange { + size_t begin; + size_t end; +}; + +using LoD = framework::LoD; + +class SplitLoDTensorOp : public framework::OperatorBase { + public: + SplitLoDTensorOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto &x = scope.FindVar(Input("X"))->Get(); + auto &mask = scope.FindVar(Input("Mask"))->Get(); + auto *out_true = + scope.FindVar(Output("OutTrue"))->GetMutable(); + auto *out_false = + scope.FindVar(Output("OutFalse"))->GetMutable(); + auto level = static_cast(Attr("level")); + auto &x_lod = x.lod(); + auto &mask_dim = mask.dims(); + + std::unique_ptr cpu_mask{new framework::LoDTensor()}; + if (platform::is_cpu_place(mask.place())) { + cpu_mask->ShareDataWith(mask); + } else if (platform::is_gpu_place(mask.place())) { +#ifdef PADDLE_WITH_CUDA + cpu_mask->CopyFrom(mask, platform::CPUPlace(), dev_ctx); +#else + PADDLE_THROW("Not supported GPU, Please compile WITH_GPU option"); +#endif + } + auto *mask_data = cpu_mask->data(); + + std::vector> copy_ranges(mask_dim[0]); + + // set out_true/out_false lod + for (size_t t = 0; t < 2; t++) { + LoD *lod = nullptr; + if (t == 0) { + lod = out_false->mutable_lod(); + } else { + lod = out_true->mutable_lod(); + } + lod->clear(); + for (size_t i = 0; i < static_cast(mask_dim[0]); i++) { + if (static_cast(mask_data[i]) == t) { + size_t start_idx = i; + auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset( + x_lod, start_idx, start_idx + 1, level); + + auto &lod_length = lod_and_offset.first; + framework::AppendLoD(lod, lod_length); + + size_t start_offset = lod_and_offset.second.first; + size_t end_offset = lod_and_offset.second.second; + copy_ranges[t].emplace_back(CopyRange{start_offset, end_offset}); + } + } + } + + for (size_t t = 0; t < 2; ++t) { + framework::LoDTensor *out; + if (t == 0) { + out = out_false; + } else { + out = out_true; + } + auto &ranges = copy_ranges[t]; + size_t height = std::accumulate( + ranges.begin(), ranges.end(), 0UL, + [](size_t a, const CopyRange &b) { return a + b.end - b.begin; }); + auto x_dim = x.dims(); + x_dim[0] = static_cast(height); + out->Resize(x_dim); + out->mutable_data(x.place(), x.type()); + size_t offset = 0; + for (auto &each_range : ranges) { + size_t len = each_range.end - each_range.begin; + if (len == 0) { + continue; + } + // out[offset: offset+len] = x[each_range.begin: each_range.end] + out->Slice(static_cast(offset), static_cast(offset + len)) + .CopyFrom(x.Slice(static_cast(each_range.begin), + static_cast(each_range.end)), + x.place(), dev_ctx); + offset += len; + } + } + } +}; + +class SplitLoDTensorOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + SplitLoDTensorOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The input LoDTensor"); + AddInput("Mask", "A bool column vector which mask the input"); + AddOutput("OutTrue", "True branch of input LoDTensor"); + AddOutput("OutFalse", "False branch of input LoDTensor"); + AddAttr("level", "(int) the specific lod level to split.") + .SetDefault(0) + .EqualGreaterThan(0); + AddComment( + R"DOC( + Split a LoDTensor with a Mask at certain level. The input LoDTensor + has 3 sequence at certain lod level. The Mask is a bool column vector, + such as [0, 1, 0] at the same level. The first and third sequence will + be send to False Output LoDTensor; whereas the second sequence will + be send to True Output LoDTensor. Please refer to MergeLoDTensorOp.)DOC"); + } +}; + +class SplitLoDTensorInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("X"), + "SplitLoDTensorOp must has input X."); + PADDLE_ENFORCE(context->HasInput("Mask"), + "SplitLoDTensorOp must has input Mask."); + PADDLE_ENFORCE(context->HasOutput("OutTrue"), + "SplitLoDTensorOp must has output OutTrue."); + PADDLE_ENFORCE(context->HasOutput("OutFalse"), + "SplitLoDTensorOp must has output OutFalse."); + + auto mask_dim = context->GetInputDim("Mask"); + PADDLE_ENFORCE_EQ(mask_dim.size(), 2); + PADDLE_ENFORCE_EQ(mask_dim[1], 1); + + context->SetOutputDim("OutTrue", context->GetInputDim("X")); + context->SetOutputDim("OutFalse", context->GetInputDim("X")); + } +}; + +class SplitLoDTensorArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("merge_lod_tensor"); + grad_op->SetInput("InTrue", OutputGrad("OutTrue")); + grad_op->SetInput("InFalse", OutputGrad("OutFalse")); + grad_op->SetInput("Mask", Input("Mask")); + grad_op->SetInput("X", Input("X")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(split_lod_tensor, ops::SplitLoDTensorOp, + ops::SplitLoDTensorOpProtoMaker, + ops::SplitLoDTensorInferShape, + ops::SplitLoDTensorArrayGradMaker); diff --git a/paddle/operators/split_op.cc b/paddle/operators/split_op.cc index 1ef314b77f0fdd395ddb0cecf8f29e97559cb7ca..275b25e96aa75fdbcb7275e272c49ea8d278d2c8 100644 --- a/paddle/operators/split_op.cc +++ b/paddle/operators/split_op.cc @@ -67,30 +67,38 @@ class SplitOpMaker : public framework::OpProtoAndCheckerMaker { public: SplitOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensor of split operator."); - AddOutput("Out", "the output tensors of split operator.").AsDuplicable(); + AddInput("X", "(Tensor) Input tensor of the split operator."); + AddOutput("Out", "(Tensor) Output tensors of the split operator.") + .AsDuplicable(); AddComment(R"DOC( - Split the input tensor into multiple sub-tensors. - Example: - Input = [[1,2], - [3,4], - [5,6]] - sections = [2,1] - axis = 0 - Output[0] = [[1,2], - [3,4]] - Output[1] = [[5,6]] +Split operator + +This operator splits the input tensor into multiple sub-tensors. + +Example: + Input = [[1,2], + [3,4], + [5,6]] + sections = [2,1] + axis = 0 + Output[0] = [[1,2], + [3,4]] + Output[1] = [[5,6]] )DOC"); AddAttr>("sections", - "the length for each" - "output along with the specify axis.") + "(vector) " + "the length of each output along the " + "specified axis.") .SetDefault(std::vector{}); AddAttr("num", - "number of the sub-tensors, it must evenly divide " + "(int, default 0)" + "Number of sub-tensors. This must evenly divide " "Input.dims()[axis]") .SetDefault(0); - AddAttr("axis", "The axis which the input will be splited on.") + AddAttr("axis", + "(int, default 0) " + "The axis which the input will be splited on.") .SetDefault(0); } }; diff --git a/paddle/operators/squared_l2_distance_op.cc b/paddle/operators/squared_l2_distance_op.cc index e360c19b47eae7fc32ae66f9e4e3873bff211b04..bec2a2c18ae8da892ee7d71f45afe53c887c0f57 100644 --- a/paddle/operators/squared_l2_distance_op.cc +++ b/paddle/operators/squared_l2_distance_op.cc @@ -59,23 +59,26 @@ class SquaredL2DistanceOpMaker : public framework::OpProtoAndCheckerMaker { SquaredL2DistanceOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "Input of SquaredL2DistanceOp."); - AddInput("Y", "Target of SquaredL2DistanceOp."); + AddInput("X", "(Tensor) Input of SquaredL2DistanceOp."); + AddInput("Y", "(Tensor) Target of SquaredL2DistanceOp."); AddOutput("sub_result", - "Buffering substraction result which " + "(Tensor) Buffering subtraction result which " "will be reused in backward.") .AsIntermediate(); - AddOutput("Out", "Squared l2 distance between input and target."); + AddOutput("Out", "(Tensor) Squared l2 distance between input and target."); AddComment(R"DOC( - SquaredL2DistanceOp will cacluate the squared L2 distance for - input and target. Number of distance value equals to the - first dimension of input. First dimension of target could be equal to - input or to 1. If the first dimension of target is 1, SquaredL2DistanceOp - will broadcast target's first dimension to input's first dimension. - You can decide whether calculate the gradient of input and target. - - Both the input X and Y can carry the LoD (Level of Details) information, - or not. But the output only shares the LoD with input X. +SquaredL2Distance operator + +This operator will cacluate the squared L2 distance for the input and +the target. Number of distance value will be equal to the first dimension +of input. First dimension of the target could be equal to the input or to 1. +If the first dimension of target is 1, the operator will broadcast target's +first dimension to input's first dimension. During backward propagation, +the user can decide whether to calculate the gradient of the input or +the target or both. + +Both the input X and Y can carry the LoD (Level of Details) information. +However, the output only shares the LoD information with input X. )DOC"); } }; diff --git a/paddle/operators/squared_l2_norm_op.cc b/paddle/operators/squared_l2_norm_op.cc index 42ad87e65a85355e1b9b927dcef9ebbb88cde717..3c10e6159f44bc8c21b1e79aefaa962c7a2b64ed 100644 --- a/paddle/operators/squared_l2_norm_op.cc +++ b/paddle/operators/squared_l2_norm_op.cc @@ -52,13 +52,13 @@ class SquaredL2NormOpMaker : public framework::OpProtoAndCheckerMaker { framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "(Tensor) The input of squared_l2_norm op."); - AddOutput("Out", "(Float) The output of squared_l2_norm op."); + AddOutput("Out", "(Scalar) The output of squared_l2_norm op."); AddComment(R"DOC( SquaredL2Norm Operator. Computes the squared L2 norm of a tensor. -Out = sum (X ** 2) +$$Out = \sum_{i} X_{i}^2$$ )DOC"); } diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index ca36ad764c8a4cb5f6c58d3ac3d9ff4a588f3200..57b99bdb3a9359bbfdbe62a6fc9afca6c4d5df9e 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -24,10 +24,16 @@ class SumOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInputs("X"), "Inputs(X) should not be null"); - auto x_dims = ctx->GetInputsDim("X"); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of SumOp should not be null."); + if (ctx->IsRuntime() && + ctx->GetOutputsVarType("Out")[0] == + framework::VarDesc::LOD_TENSOR_ARRAY) { + return; // skip runtime infershape when is tensor array; + } + auto x_dims = ctx->GetInputsDim("X"); size_t N = x_dims.size(); PADDLE_ENFORCE_GT(N, 1, "Input tensors count should > 1."); @@ -39,19 +45,47 @@ class SumOp : public framework::OperatorWithKernel { ctx->SetOutputDim("Out", in_dim); ctx->ShareLoD("X", /*->*/ "Out"); } + + protected: + framework::OpKernelType GetKernelType( + const framework::ExecutionContext& ctx) const override { + auto x_vars = ctx.MultiInputVar("X"); + if (x_vars[0]->IsType()) { + return framework::OpKernelType( + framework::ToDataType(x_vars[0]->Get().type()), + ctx.device_context()); + } else if (x_vars[0]->IsType()) { + return framework::OpKernelType( + framework::ToDataType( + x_vars[0]->Get().value().type()), + ctx.device_context()); + } else if (x_vars[0]->IsType()) { + auto& array = x_vars[0]->Get(); + for (auto& each : array) { + if (each.numel() != 0) { + return framework::OpKernelType(framework::ToDataType(each.type()), + ctx.device_context()); + } + } + } + PADDLE_THROW("Unexpected branch. Input type is %s", + x_vars[0]->Type().name()); + } }; class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: SumOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of sum operator.").AsDuplicable(); - AddOutput("Out", "the output tensor of sum operator."); + AddInput("X", "(vector) The input tensors of sum operator.") + .AsDuplicable(); + AddOutput("Out", "(Tensor) The output tensor of sum operator."); AddComment(R"DOC( -Sum the input tensors. +Sum operator. -All the inputs can carry the LoD (Level of Details) information, -or not. But the output only shares the LoD with the first input. +This operators sums the input tensors. All the inputs can carry the +LoD (Level of Details) information. However, the output only shares +the LoD information with the first input. )DOC"); } }; @@ -61,18 +95,33 @@ class SumOpVarTypeInference : public framework::VarTypeInference { void operator()(const framework::OpDescBind& op_desc, framework::BlockDescBind* block) const override { auto& inputs = op_desc.Input("X"); - auto default_var_type = framework::VarDesc::SELECTED_ROWS; + auto var_type = framework::VarDesc::SELECTED_ROWS; bool any_input_is_lod_tensor = std::any_of( inputs.begin(), inputs.end(), [block](const std::string& name) { - return block->Var(name)->GetType() == framework::VarDesc::LOD_TENSOR; + return block->FindRecursiveOrCreateVar(name)->GetType() == + framework::VarDesc::LOD_TENSOR; }); - if (any_input_is_lod_tensor) { - default_var_type = framework::VarDesc::LOD_TENSOR; + + auto is_tensor_array = [block](const std::string& name) { + return block->FindRecursiveOrCreateVar(name)->GetType() == + framework::VarDesc::LOD_TENSOR_ARRAY; + }; + + bool any_input_is_tensor_array = + std::any_of(inputs.begin(), inputs.end(), is_tensor_array); + bool all_inputs_are_tensor_array = + std::all_of(inputs.begin(), inputs.end(), is_tensor_array); + + if (any_input_is_tensor_array) { + PADDLE_ENFORCE(all_inputs_are_tensor_array); + var_type = framework::VarDesc::LOD_TENSOR_ARRAY; + } else if (any_input_is_lod_tensor) { + var_type = framework::VarDesc::LOD_TENSOR; } auto out_var_name = op_desc.Output("Out").front(); - block->Var(out_var_name)->SetType(default_var_type); + block->FindRecursiveOrCreateVar(out_var_name)->SetType(var_type); } }; diff --git a/paddle/operators/sum_op.h b/paddle/operators/sum_op.h index f2f2c67bc395ea245798b537144dd88a816f4a85..4ca15611392b3117aa6c92cba95911eb8bebeb15 100644 --- a/paddle/operators/sum_op.h +++ b/paddle/operators/sum_op.h @@ -11,6 +11,7 @@ limitations under the License. */ #pragma once #include "paddle/framework/eigen.h" +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/math/math_function.h" #include "paddle/operators/math/selected_rows_functor.h" @@ -28,37 +29,43 @@ using EigenVector = framework::EigenVector; template class SumKernel : public framework::OpKernel { public: - void Compute(const framework::ExecutionContext& context) const override { - auto& in_vars = context.MultiInputVar("X"); + void Compute(const framework::ExecutionContext &context) const override { + auto in_vars = context.MultiInputVar("X"); int N = in_vars.size(); auto out_var = context.OutputVar("Out"); + bool in_place = out_var == in_vars[0]; + if (out_var->IsType()) { - auto* out = context.Output("Out"); + auto *out = context.Output("Out"); out->mutable_data(context.GetPlace()); auto result = EigenVector::Flatten(*out); - math::SetConstant constant_functor; - constant_functor(context.device_context(), out, 0.0); + if (!in_place) { + math::SetConstant constant_functor; + constant_functor(context.device_context(), out, 0.0); + } math::SelectedRowsAddToTensor functor; auto place = context.GetEigenDevice(); - for (int i = 0; i < N; i++) { + // If in_place, just skip the first tensor + for (int i = in_place ? 1 : 0; i < N; i++) { if (in_vars[i]->IsType()) { - auto& in_t = in_vars[i]->Get(); + auto &in_t = in_vars[i]->Get(); auto in = EigenVector::Flatten(in_t); result.device(place) = result + in; } else if (in_vars[i]->IsType()) { - auto& in_t = in_vars[i]->Get(); + auto &in_t = in_vars[i]->Get(); functor(context.device_context(), in_t, out); } else { PADDLE_THROW("Variable type must be LoDTensor/SelectedRows."); } } } else if (out_var->IsType()) { - auto* out = context.Output("Out"); - auto* out_value = out->mutable_value(); + PADDLE_ENFORCE(!in_place, "SelectedRows not support inplace sum now"); + auto *out = context.Output("Out"); + auto *out_value = out->mutable_value(); // Runtime InferShape size_t first_dim = 0; @@ -82,9 +89,36 @@ class SumKernel : public framework::OpKernel { offset, out); offset += in_vars[i]->Get().value().numel(); } + } else if (out_var->IsType()) { + auto &out_array = *out_var->GetMutable(); + for (size_t i = in_place ? 1 : 0; i < in_vars.size(); ++i) { + PADDLE_ENFORCE(in_vars[i]->IsType(), + "Only support all inputs are TensorArray"); + auto &in_array = in_vars[i]->Get(); + + for (size_t i = 0; i < in_array.size(); ++i) { + if (in_array[i].numel() != 0) { + if (i >= out_array.size()) { + out_array.resize(i + 1); + } + if (out_array[i].numel() == 0) { + out_array[i].CopyFrom(in_array[i], in_array[i].place(), + context.device_context()); + out_array[i].set_lod(in_array[i].lod()); + } else { + PADDLE_ENFORCE(out_array[i].lod() == in_array[i].lod()); + auto in = EigenVector::Flatten(in_array[i]); + auto result = EigenVector::Flatten(out_array[i]); + result.device(context.GetEigenDevice()) = result + in; + } + } + } + } + } else { + PADDLE_THROW("Unexpected branch, output variable type is %s", + out_var->Type().name()); } } }; - } // namespace operators } // namespace paddle diff --git a/paddle/operators/tensor_array_read_write_op.cc b/paddle/operators/tensor_array_read_write_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..62e15604c47f25c458abc69ecd1cabf964de39bb --- /dev/null +++ b/paddle/operators/tensor_array_read_write_op.cc @@ -0,0 +1,189 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ +#include "paddle/operators/array_operator.h" + +namespace paddle { +namespace operators { + +class WriteToArrayOp : public ArrayOp { + public: + WriteToArrayOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x != nullptr, "X must be set"); + auto &x_tensor = x->Get(); + size_t offset = GetOffset(scope, dev_ctx); + auto *out = + scope.FindVar(Output("Out"))->GetMutable(); + if (offset >= out->size()) { + out->resize(offset + 1); + } + auto *out_tensor = &out->at(offset); + out_tensor->CopyFrom(x_tensor, dev_ctx.GetPlace(), dev_ctx); + out_tensor->set_lod(x_tensor.lod()); + } +}; + +class WriteToArrayOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + WriteToArrayOpProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(LoDTensor) the tensor will be written to tensor array"); + AddInput( + "I", + "(Tensor) the subscript index in tensor array. The number of element " + "should be 1"); + AddOutput("Out", "(TensorArray) the tensor array will be written"); + AddComment(R"DOC(Write a LoDTensor to a LoDTensor array. + +Assume T is LoDTensor, i is the subscript of the array, and A is the array. The +equation is + +A[i] = T +)DOC"); + } +}; + +class WriteToArrayInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInput("I"), "Must set the subscript index"); + PADDLE_ENFORCE_EQ(framework::product(context->GetInputDim("I")), 1, + "The number of element of subscript index must be 1"); + PADDLE_ENFORCE(context->HasInput("X"), NotHasXError()); + PADDLE_ENFORCE(context->HasOutput("Out"), NotHasOutError()); + context->SetOutputDim("Out", context->GetInputDim("X")); + } + + protected: + virtual const char *NotHasXError() const { return "Must set the lod tensor"; } + + virtual const char *NotHasOutError() const { + return "Must set the lod tensor array"; + } +}; + +class WriteToArrayInferVarType : public framework::VarTypeInference { + public: + void operator()(const framework::OpDescBind &op_desc, + framework::BlockDescBind *block) const override { + for (auto &out_var : op_desc.OutputArgumentNames()) { + VLOG(10) << "Set Variable " << out_var << " as LOD_TENSOR_ARRAY"; + block->FindRecursiveOrCreateVar(out_var)->SetType( + framework::VarDesc::LOD_TENSOR_ARRAY); + } + } +}; + +class ReadFromArrayOp : public ArrayOp { + public: + ReadFromArrayOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : ArrayOp(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + auto *x = scope.FindVar(Input("X")); + PADDLE_ENFORCE(x != nullptr, "X must be set"); + auto &x_array = x->Get(); + auto *out = scope.FindVar(Output("Out")); + PADDLE_ENFORCE(out != nullptr, "Out must be set"); + auto *out_tesnor = out->GetMutable(); + size_t offset = GetOffset(scope, dev_ctx); + PADDLE_ENFORCE_LT(offset, x_array.size()); + out_tesnor->CopyFrom(x_array[offset], dev_ctx.GetPlace(), dev_ctx); + out_tesnor->set_lod(x_array[offset].lod()); + } +}; + +class ReadFromArrayProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ReadFromArrayProtoMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "(TensorArray) the array will be read from."); + AddInput("I", + "(Tensor) the subscript index in tensor array. The number of " + "element should be 1"); + AddOutput("Out", "(LoDTensor) the tensor will be read from."); + AddComment(R"DOC(Read a LoDTensor from a LoDTensor Array + +Assume T is LoDTensor, i is th e subscript of the array, and A is the array. The +equation is + +T = A[i] +)DOC"); + } +}; + +class ReadFromArrayInferShape : public WriteToArrayInferShape { + protected: + const char *NotHasXError() const override { + return "The input array X must be set"; + } + const char *NotHasOutError() const override { + return "The output tensor out must be set"; + } +}; + +class WriteToArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("read_from_array"); + grad_op->SetInput("I", Input("I")); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +class ReadFromArrayGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("write_to_array"); + grad_op->SetInput("I", Input("I")); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(write_to_array, ops::WriteToArrayOp, + ops::WriteToArrayInferShape, ops::WriteToArrayOpProtoMaker, + ops::WriteToArrayGradMaker, ops::WriteToArrayInferVarType); +REGISTER_OPERATOR(read_from_array, ops::ReadFromArrayOp, + ops::ReadFromArrayInferShape, ops::ReadFromArrayProtoMaker, + ops::ReadFromArrayGradMaker); diff --git a/paddle/operators/top_k_op.cc b/paddle/operators/top_k_op.cc index ac9257259548dc4223efac239d8362f69366850c..16ae925eb5cab1c05f3bc376972cabadc4367d20 100644 --- a/paddle/operators/top_k_op.cc +++ b/paddle/operators/top_k_op.cc @@ -48,20 +48,20 @@ class TopkOpMaker : public framework::OpProtoAndCheckerMaker { public: TopkOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The input of Topk op"); - AddOutput("Out", "The output tensor of Topk op"); - AddOutput("Indices", "The indices of Topk elements of input"); - AddComment( - R"DOC(If the input is a vector (1d tensor), - finds the k largest entries in the vector - and outputs their values and indices as vectors. - Thus values[j] is the j-th largest entry in input, - and its index is indices[j]. + AddInput("X", "(Tensor) The input of Topk op"); + AddOutput("Out", "(Tensor) The output tensor of Topk op"); + AddOutput("Indices", "(Tensor) The indices of Topk elements of input"); + AddComment(R"DOC( +Top K operator - For matrices, computes the top k entries in each row. )DOC"); +If the input is a vector (1d tensor), this operator finds the k largest +entries in the vector and outputs their values and indices as vectors. +Thus values[j] is the j-th largest entry in input, and its index is indices[j]. + +For matrices, this operator computes the top k entries in each row. )DOC"); AddAttr("k", - "Number of top elements to look for along the last " - "dimension (along each row for matrices).") + "(int, default 1) Number of top elements to look for along " + "the last dimension (along each row for matrices).") .SetDefault(1); } }; diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc index d785e57c830439ad80005d9a3d4bb77faf1ae1b9..94de3d5069017a7ca818e246ad574c4db92d8006 100644 --- a/paddle/operators/transpose_op.cc +++ b/paddle/operators/transpose_op.cc @@ -32,7 +32,7 @@ class TransposeOp : public framework::OperatorWithKernel { size_t axis_size = axis.size(); PADDLE_ENFORCE_EQ(x_rank, axis_size, - "the input tensor's rank(%d) " + "The input tensor's rank(%d) " "should be equal to the axis's size(%d)", x_rank, axis_size); @@ -64,12 +64,14 @@ class TransposeOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "(Tensor)The output tensor"); AddAttr>( "axis", - "(vector)a list of values, and the size of the list should be " + "(vector)A list of values, and the size of the list should be " "the same with the input tensor rank, the tensor will " "permute the axes according the the values given"); AddComment(R"DOC( -The Tensor will be permuted according to the axis values given. -The op is very much like the numpy.transpose function in python +Transpose Operator. + +The input tensor will be permuted according to the axis values given. +The op functions similar to how numpy.transpose works in python. For example: >> input = numpy.arange(6).reshape((2,3)) >> input @@ -83,6 +85,7 @@ For example: [2, 5]]) So, given a input tensor of shape(N, C, H, W) and the axis is {0, 2, 3, 1}, the output tensor shape will be (N, H, W, C) + )DOC"); } }; diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index 82f9b8fbf1094bde1def83b9a1c464207b7e4669..7975efc7cf134aaf591385a6866254a9c5f2a0bb 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -63,9 +63,11 @@ class UniformRandomOp : public framework::OperatorWithKernel { } protected: - framework::DataType IndicateDataType( + framework::OpKernelType GetKernelType( const framework::ExecutionContext& ctx) const override { - return static_cast(ctx.Attr("data_type")); + return framework::OpKernelType( + static_cast(ctx.Attr("data_type")), + ctx.device_context()); } }; @@ -74,18 +76,30 @@ class UniformRandomOpMaker : public framework::OpProtoAndCheckerMaker { UniformRandomOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : framework::OpProtoAndCheckerMaker(proto, op_checker) { - AddOutput("Out", "The output tensor of uniform random op"); - AddComment(R"DOC(Uniform random operator. -Used to initialize tensor with uniform random generator. + AddOutput("Out", "(Tensor) The output tensor of uniform random op"); + AddComment(R"DOC( +Uniform random operator. + +This operator initializes a tensor with random values sampled from a +uniform distribution. + )DOC"); - AddAttr>("shape", "the dimension of random tensor"); - AddAttr("min", "Minimum value of uniform random").SetDefault(-1.0f); - AddAttr("max", "Maximun value of uniform random").SetDefault(1.0f); + AddAttr>("shape", + "(vector) The shape of the output tensor"); + AddAttr("min", + "(float, default -1.0) " + "Minimum value of uniform random") + .SetDefault(-1.0f); + AddAttr("max", + "(float, default 1.0) " + "Maximun value of uniform random") + .SetDefault(1.0f); AddAttr("seed", - "Random seed of uniform random. " - "0 means generate a seed by system") + "(int, default 0) " + "Random seed used for generating samples. " + "0 means use a seed generated by the system.") .SetDefault(0); - AddAttr("data_type", "output tensor data type") + AddAttr("data_type", "(int, default 5(FP32)) Output tensor data type") .SetDefault(framework::DataType::FP32); } }; diff --git a/paddle/operators/while_op.cc b/paddle/operators/while_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..4ca6c8507a48507fd29a9c9acae2bdf36ed936ee --- /dev/null +++ b/paddle/operators/while_op.cc @@ -0,0 +1,197 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include "paddle/framework/executor.h" +#include "paddle/framework/op_registry.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace operators { + +using StepScopeVar = std::vector; +using LoDTensor = framework::LoDTensor; + +constexpr char kStepBlock[] = "step_block"; +constexpr char kCondition[] = "Condition"; +constexpr char kStepScopes[] = "StepScopes"; +constexpr char kParamGrads[] = "X@Grad"; +constexpr char kParameters[] = "X"; + +class WhileOp : public framework::OperatorBase { + public: + WhileOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + PADDLE_ENFORCE_NOT_NULL(scope.FindVar(Input(kCondition))); + auto &cond = scope.FindVar(Input(kCondition))->Get(); + PADDLE_ENFORCE_EQ(cond.dims(), paddle::framework::make_ddim({1})); + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + auto step_scopes = + scope.FindVar(Output(kStepScopes))->GetMutable(); + + while (cond.data()[0]) { + auto ¤t_scope = scope.NewScope(); + step_scopes->push_back(¤t_scope); + + executor.Run(*program, ¤t_scope, block->ID(), + false /*create_local_scope*/); + } + } +}; + +class WhileOpMaker : public framework::OpProtoAndCheckerMaker { + public: + WhileOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput(kParameters, + "A set of variables, which are required by operators inside the " + "block of While Op.") + .AsDuplicable(); + AddInput( + kCondition, + "(Bool) An scalar. When it's False, the While Op will be terminated.") + .AsDuplicable(); + AddOutput("Out", + "A set of variables, which will be assigned with values " + "generated by perators inside the block of While Op.") + .AsDuplicable(); + AddOutput(kStepScopes, + "(StepScopeVar) A vector of local scope, which size equals the " + "step number of While Op. The i'th scope storages temporary " + "variables generated in the i'th step."); + AddAttr(kStepBlock, + "The step block inside WhileOp"); + AddComment(R"DOC( +)DOC"); + } +}; + +class WhileGradOp : public framework::OperatorBase { + public: + WhileGradOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::DeviceContext &dev_ctx) const override { + // PADDLE_ENFORCE(...) + + framework::Executor executor(dev_ctx); + auto *block = Attr(kStepBlock); + auto *program = block->Program(); + + auto *step_scopes = + scope.FindVar(Input(kStepScopes))->GetMutable(); + + for (auto cur_scope_iter = step_scopes->rbegin(); + cur_scope_iter != step_scopes->rend(); ++cur_scope_iter) { + executor.Run(*program, *cur_scope_iter, block->ID(), false); + + auto &pg_names = Outputs(kParamGrads); + auto &p_names = Inputs(kParameters); + PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size()); + for (size_t prog_id = 0; prog_id < pg_names.size(); ++prog_id) { + auto inside_grad_name = framework::GradVarName(p_names[prog_id]); + + // // TODO(tonyyang-savil: Not sure we need the following + // // If does not compute gradient of that variable inside rnn, + // just + // // continue + // if (local_var_names.find(inside_grad_name) == + // local_var_names.end()) { + // continue; + // } + + // zero gradient variable in step 0 + if (cur_scope_iter == step_scopes->rbegin()) { + auto *var = (*cur_scope_iter)->FindVar(inside_grad_name); + PADDLE_ENFORCE_NOT_NULL(var); + if (var->IsType()) { + auto &inside_tensor = var->Get(); + framework::AttributeMap attrs; + attrs["data_type"] = framework::ToDataType(inside_tensor.type()); + attrs["shape"] = framework::vectorize2int(inside_tensor.dims()); + attrs["value"] = 0.0f; + + auto zero_op = framework::OpRegistry::CreateOp( + "fill_constant", {}, {{"Out", {pg_names[prog_id]}}}, attrs); + zero_op->Run(scope, dev_ctx); + } + } + + // sum gradient + auto *outside_var = scope.FindVar(pg_names[prog_id]); + PADDLE_ENFORCE_NOT_NULL(outside_var); + auto &outside_tensor = *outside_var->GetMutable(); + + std::string result_var_name; + auto *local_result_var = (*cur_scope_iter)->Var(&result_var_name); + auto &local_result_tensor = + *local_result_var->GetMutable(); + + local_result_tensor.ShareDataWith(outside_tensor); + + auto sum_op = framework::OpRegistry::CreateOp( + "sum", {{"X", {result_var_name, inside_grad_name}}}, + {{"Out", {result_var_name}}}, {}); + sum_op->Run(**cur_scope_iter, dev_ctx); + } + } + } +}; + +class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + virtual std::unique_ptr Apply() const { + auto *grad = new framework::OpDescBind(); + grad->SetType("while_grad"); + for (auto &input_param : this->InputNames()) { + grad->SetInput(input_param, this->Input(input_param)); + grad->SetOutput(framework::GradVarName(input_param), + this->InputGrad(input_param)); + } + + for (auto &output_param : this->OutputNames()) { + grad->SetInput(output_param, this->Output(output_param)); + if (output_param != kStepScopes) { + grad->SetInput(framework::GradVarName(output_param), + this->OutputGrad(output_param)); + } + } + grad->SetAttrMap(this->Attrs()); + grad->SetBlockAttr(kStepBlock, *grad_block_[0]); + + return std::unique_ptr(grad); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(while, paddle::operators::WhileOp, + paddle::operators::WhileOpMaker, + paddle::operators::WhileGradOpDescMaker); diff --git a/paddle/optimizer/CMakeLists.txt b/paddle/optimizer/CMakeLists.txt index 926fee47e1f86efa60dc40a2727edb06499bec4f..25fc35311fc63988c64a445d72fc6255e49e8d4b 100644 --- a/paddle/optimizer/CMakeLists.txt +++ b/paddle/optimizer/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories(${CMAKE_CURRENT_BINARY_DIR}) - set(OPITMIZER_SRCS adadelta_optimizer.cc adagrad_optimizer.cc @@ -9,11 +7,6 @@ set(OPITMIZER_SRCS sgd_optimizer.cc ) -add_library(paddle_optimizer STATIC ${OPITMIZER_SRCS}) -add_dependencies(paddle_optimizer paddle_proto ${external_project_dependencies}) - - -if(WITH_TESTING) - add_simple_unittest(serialization_test) - add_simple_unittest(parameter_optimizer_test) -endif() +cc_library(paddle_optimizer STATIC SRCS ${OPITMIZER_SRCS} DEPS paddle_proto glog) +cc_test(serialization_test SRCS serialization_test.cc DEPS paddle_proto) +cc_test(parameter_optimizer_test SRCS parameter_optimizer_test.cc DEPS paddle_optimizer) diff --git a/paddle/optimizer/adadelta_optimizer.cc b/paddle/optimizer/adadelta_optimizer.cc index 34913c405075ed72af30ed056f74e8b4d7482488..5cc7c47d4486c3d149c37fd6e312780f3d44eda8 100644 --- a/paddle/optimizer/adadelta_optimizer.cc +++ b/paddle/optimizer/adadelta_optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include "adadelta_optimizer.h" #include #include diff --git a/paddle/optimizer/adadelta_optimizer.h b/paddle/optimizer/adadelta_optimizer.h index bc634ee46d60abc9ffc4a31abac5c2f8edaf7aba..6aab1ad553b15ebbd2d04c9323c5e56e1b8f60f5 100644 --- a/paddle/optimizer/adadelta_optimizer.h +++ b/paddle/optimizer/adadelta_optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include "parameter_optimizer.h" diff --git a/paddle/optimizer/adagrad_optimizer.cc b/paddle/optimizer/adagrad_optimizer.cc index d915ffb8705eaa96bc96b8071a2c534d4d472273..c981996bab1b2e7ae5d6e2d858a73efde12e32f3 100644 --- a/paddle/optimizer/adagrad_optimizer.cc +++ b/paddle/optimizer/adagrad_optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include #include "adagrad_optimizer.h" diff --git a/paddle/optimizer/adagrad_optimizer.h b/paddle/optimizer/adagrad_optimizer.h index b2935f8aff87f710f508c5c5757dd36526ca63f9..447b7c7547d5bad7436df6f3b3582b4a219f08c8 100644 --- a/paddle/optimizer/adagrad_optimizer.h +++ b/paddle/optimizer/adagrad_optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include "parameter_optimizer.h" diff --git a/paddle/optimizer/adam_optimizer.cc b/paddle/optimizer/adam_optimizer.cc index 18e5896a22dc8a3c6292293fffc36ca9e3737b4c..6dc2d749708d0e2a7f36734d89eec30d4576842e 100644 --- a/paddle/optimizer/adam_optimizer.cc +++ b/paddle/optimizer/adam_optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include "adam_optimizer.h" #include diff --git a/paddle/optimizer/adam_optimizer.h b/paddle/optimizer/adam_optimizer.h index d25cdc0731f65e9875d2fbf67783cce62d88af60..37ab53afc37a5f749a2909de12c7871ed926583f 100644 --- a/paddle/optimizer/adam_optimizer.h +++ b/paddle/optimizer/adam_optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include "parameter_optimizer.h" diff --git a/paddle/optimizer/optimizer.cc b/paddle/optimizer/optimizer.cc index a2af139d012433214b825bd68289708098b76da8..faa23764522cef03bae1359adbf58d10ee7809ac 100644 --- a/paddle/optimizer/optimizer.cc +++ b/paddle/optimizer/optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include "optimizer.h" #include #include @@ -6,8 +20,8 @@ #include "parameter_optimizer.h" -using namespace paddle; -using namespace paddle::optimizer; +using paddle::optimizer::ParameterOptimizer; +using paddle::optimizer::Tensor; template struct EnumToType {}; @@ -15,22 +29,21 @@ struct EnumToType {}; template struct TypeToEnum {}; -#define MATCH_ENUM_TYPE(TYPE, ENUM) \ - template <> \ - struct TypeToEnum { \ - static paddle_element_type v() { return ENUM; }; \ - static constexpr TYPE value = ENUM; \ - }; \ - template <> \ - struct EnumToType { \ - typedef TYPE Type; \ +#define MATCH_ENUM_TYPE(TYPE, ENUM) \ + template <> \ + struct TypeToEnum { \ + static paddle_element_type v() { return ENUM; } \ + static constexpr TYPE value = ENUM; \ + }; \ + template <> \ + struct EnumToType { \ + typedef TYPE Type; \ } MATCH_ENUM_TYPE(int32_t, PADDLE_ELEMENT_TYPE_INT32); MATCH_ENUM_TYPE(uint32_t, PADDLE_ELEMENT_TYPE_UINT32); MATCH_ENUM_TYPE(int64_t, PADDLE_ELEMENT_TYPE_INT64); MATCH_ENUM_TYPE(uint64_t, PADDLE_ELEMENT_TYPE_UINT64); -// TODO(zhihong): only implement below type, need to fix MATCH_ENUM_TYPE(float, PADDLE_ELEMENT_TYPE_FLOAT32); MATCH_ENUM_TYPE(double, PADDLE_ELEMENT_TYPE_FLOAT64); diff --git a/paddle/optimizer/optimizer.h b/paddle/optimizer/optimizer.h index aabf7a458dd30092ed1e522c4d88c6cfe63fcce1..e6fa12a4d250ccb078358704b0131942ea6ab039 100644 --- a/paddle/optimizer/optimizer.h +++ b/paddle/optimizer/optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include diff --git a/paddle/optimizer/parameter_optimizer.cc b/paddle/optimizer/parameter_optimizer.cc index db0714635f9366b0404019688daf4708b4a0052f..da92c2d01cc2a27d1fadd51a338d23b01e0cb0bc 100644 --- a/paddle/optimizer/parameter_optimizer.cc +++ b/paddle/optimizer/parameter_optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include #include "adadelta_optimizer.h" #include "adagrad_optimizer.h" diff --git a/paddle/optimizer/parameter_optimizer.h b/paddle/optimizer/parameter_optimizer.h index 8319f84e1b820adf5cc0006045f2e13dffa91797..99d0416e751c4ca6695d6ed77396e18d48fc86b8 100644 --- a/paddle/optimizer/parameter_optimizer.h +++ b/paddle/optimizer/parameter_optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include diff --git a/paddle/optimizer/parameter_optimizer_test.cpp b/paddle/optimizer/parameter_optimizer_test.cc similarity index 97% rename from paddle/optimizer/parameter_optimizer_test.cpp rename to paddle/optimizer/parameter_optimizer_test.cc index c88fa11748716693355042d1784b33d7cfb616f1..f29e5317120642e3790a6f6c1976bdda67093a0c 100644 --- a/paddle/optimizer/parameter_optimizer_test.cpp +++ b/paddle/optimizer/parameter_optimizer_test.cc @@ -85,7 +85,7 @@ public: for (size_t i = 0; i < opts_.size(); ++i) { int s = 0; float* newp = (float*)opts_[i]->get_weight(&s); - EXPECT_EQ(s, kSize); + EXPECT_EQ(static_cast(s), kSize); for (size_t j = 0; j < kSize; ++j) { EXPECT_EQ(newp[j], (*p)[j]); } @@ -110,7 +110,7 @@ public: int s = 0; float* newp = (float*)opts_[i]->get_weight(&s); - EXPECT_EQ(s, kSize); + EXPECT_EQ(static_cast(s), kSize); for (size_t j = 0; j < kSize; ++j) { EXPECT_EQ(newp[j], (*p)[j]); } diff --git a/paddle/optimizer/serialization_test.cpp b/paddle/optimizer/serialization_test.cc similarity index 100% rename from paddle/optimizer/serialization_test.cpp rename to paddle/optimizer/serialization_test.cc diff --git a/paddle/optimizer/sgd_optimizer.cc b/paddle/optimizer/sgd_optimizer.cc index 1090419083c8b8cf60eca02791ef673287f4a9a4..c150144ac24b8375d08691a98be680b6bf5d1e7f 100644 --- a/paddle/optimizer/sgd_optimizer.cc +++ b/paddle/optimizer/sgd_optimizer.cc @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #include "sgd_optimizer.h" #include "serialization.h" diff --git a/paddle/optimizer/sgd_optimizer.h b/paddle/optimizer/sgd_optimizer.h index 6e1a0f0d3f9ecfeb51ccb355d65985a2e6388fb0..0b1da0aa27d98e8d6a8d9fd7a1ebe355acb2a1f4 100644 --- a/paddle/optimizer/sgd_optimizer.h +++ b/paddle/optimizer/sgd_optimizer.h @@ -1,3 +1,17 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + #pragma once #include "parameter_optimizer.h" @@ -15,7 +29,6 @@ public: nesterov_(n) { if (momentum_ != 0.0) { size_t size = parameter->size(); - // TODO: fix it with align aware allocator bind to Tensor momentums_ = new Tensor(size); } } diff --git a/paddle/platform/call_once.h b/paddle/platform/call_once.h new file mode 100644 index 0000000000000000000000000000000000000000..d9f49527dcf150fcb35d3af512088f75dec0b5c6 --- /dev/null +++ b/paddle/platform/call_once.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include + +namespace paddle { +namespace platform { + +/* + The current implementation of std::call_once has a bug described in + https://stackoverflow.com/questions/41717579/stdcall-once-hangs-on-second-call-after-callable-threw-on-first-call. + This is likely caused by a deeper bug of pthread_once, which is discussed in + https://patchwork.ozlabs.org/patch/482350/ + + This wrap is a hack to avoid this bug. +*/ +template +inline void call_once(std::once_flag& flag, Callable&& f, Args&&... args) { + bool good = false; + std::exception ex; + std::call_once(flag, + [&](Args&&... args) { + try { + f(args...); + good = true; + } catch (const std::exception& e) { + ex = e; + } catch (...) { + ex = std::runtime_error("excption caught in call_once"); + } + }, + args...); + if (!good) { + throw std::exception(ex); + } +} + +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 36450e926891342f37424447703781a33c1190ae..7afcdfce9371e29aad968a1729931173fb2309b5 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -124,6 +124,11 @@ void CUDADeviceContext::Wait() const { PADDLE_ENFORCE(cudaStreamSynchronize(stream_)); } +void CUDADeviceContext::Finish() const { + Wait(); + PADDLE_ENFORCE(cudaGetLastError()); +} + Eigen::GpuDevice* CUDADeviceContext::eigen_device() const { return eigen_device_.get(); } diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index ef5f19214d9ccb23b9c946bee28cb764122bd7cd..526d089e35da9c9f89a3852095ad3a4c82d4d85d 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -46,6 +46,8 @@ class DeviceContext { DeviceType* GetEigenDevice() const; virtual void Wait() const {} + + virtual void Finish() const {} }; class CPUDeviceContext : public DeviceContext { @@ -77,6 +79,9 @@ class CUDADeviceContext : public DeviceContext { /*! \brief Wait for all operations completion in the stream. */ void Wait() const override; + /*! \brief Check potential errors for the cuda kernel calls. */ + void Finish() const override; + /*! \brief Return place in the device context. */ Place GetPlace() const override; diff --git a/paddle/platform/dynload/nccl.h b/paddle/platform/dynload/nccl.h index 0618c7414fd1235e81ee9d92a3a07b53d6ad6ebc..981b2ab258a34ce92f02ee12b5957f88ba61d1c0 100644 --- a/paddle/platform/dynload/nccl.h +++ b/paddle/platform/dynload/nccl.h @@ -17,6 +17,7 @@ #include #include #include +#include "paddle/platform/call_once.h" #include "paddle/platform/dynload/dynamic_loader.h" namespace paddle { @@ -27,18 +28,18 @@ extern std::once_flag nccl_dso_flag; extern void* nccl_dso_handle; #ifdef PADDLE_USE_DSO -#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> decltype(__name(args...)) { \ - using nccl_func = decltype(__name(args...)) (*)(Args...); \ - std::call_once(nccl_dso_flag, \ - paddle::platform::dynload::GetNCCLDsoHandle, \ - &nccl_dso_handle); \ - void* p_##__name = dlsym(nccl_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ +#define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using nccl_func = decltype(__name(args...)) (*)(Args...); \ + platform::call_once(nccl_dso_flag, \ + paddle::platform::dynload::GetNCCLDsoHandle, \ + &nccl_dso_handle); \ + void* p_##__name = dlsym(nccl_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ extern DynLoad__##__name __name #else #define DECLARE_DYNAMIC_LOAD_NCCL_WRAP(__name) \ diff --git a/paddle/platform/transform.h b/paddle/platform/transform.h index f196868c725cbb91b3df710260c5b60f14d53f37..bb9d59ec0a18ce013632f128c9b5d230255f1ac4 100644 --- a/paddle/platform/transform.h +++ b/paddle/platform/transform.h @@ -49,8 +49,6 @@ struct Transform { template void operator()(const DeviceContext& context, InputIter first, InputIter last, OutputIter result, UnaryOperation op) { - auto place = context.GetPlace(); - PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first, last, result, op); } @@ -59,8 +57,6 @@ struct Transform { void operator()(const DeviceContext& context, InputIter1 first1, InputIter1 last1, InputIter2 first2, OutputIter result, BinaryOperation op) { - auto place = context.GetPlace(); - PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first1, last1, first2, result, op); } }; diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index dcae426c7e231757d796c5a84cc5a1c2b0d6763b..5a1ff9b7976abbe4a37f8366181d9d1ae78ea4a0 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -97,6 +97,15 @@ namespace pybind { using namespace paddle::framework; // NOLINT +template +static py::bytes SerializeMessage(T &self) { + // Check IsInitialized in Python + std::string retv; + PADDLE_ENFORCE(self.Proto()->SerializePartialToString(&retv), + "Cannot serialize message"); + return retv; +} + // Bind Methods void BindProgramDesc(py::module &m) { py::class_(m, "ProgramDesc", "") @@ -132,17 +141,7 @@ void BindProgramDesc(py::module &m) { .def("block", &ProgramDescBind::MutableBlock, py::return_value_policy::reference) .def("num_blocks", &ProgramDescBind::Size) - .def("serialize_to_string", - [](ProgramDescBind &program_desc) -> py::bytes { - const ProgramDesc *desc = program_desc.Proto(); - PADDLE_ENFORCE(desc->IsInitialized(), - "ProgramDesc has not been initialized."); - std::string res; - PADDLE_ENFORCE( - desc->SerializeToString(&res), - "Serialize ProgramDesc Error. This could be a bug of Paddle."); - return res; - }) + .def("serialize_to_string", SerializeMessage) .def("parse_from_string", [](ProgramDescBind &program_desc, const std::string &data) { ProgramDesc *desc = program_desc.Proto(); @@ -181,16 +180,7 @@ void BindBlockDesc(py::module &m) { py::return_value_policy::reference) .def("op_size", &BlockDescBind::OpSize) .def("op", &BlockDescBind::Op, py::return_value_policy::reference) - .def("serialize_to_string", [](BlockDescBind &block_desc) -> py::bytes { - const BlockDesc *desc = block_desc.Proto(); - PADDLE_ENFORCE(desc->IsInitialized(), - "BlockDesc has not been initialized."); - std::string res; - PADDLE_ENFORCE( - desc->SerializeToString(&res), - "Serialize BlockDesc Error. This could be a bug of Paddle."); - return res; - }); + .def("serialize_to_string", SerializeMessage); } void BindVarDsec(py::module &m) { @@ -219,17 +209,7 @@ void BindVarDsec(py::module &m) { .def("set_lod_level", &VarDescBind::SetLoDLevel) .def("type", &VarDescBind::GetType) .def("set_type", &VarDescBind::SetType) - .def("serialize_to_string", - [](VarDescBind &var_desc) -> py::bytes { - const VarDesc *desc = var_desc.Proto(); - PADDLE_ENFORCE(desc->IsInitialized(), - "VarDesc has not been initialized."); - std::string res; - PADDLE_ENFORCE( - desc->SerializeToString(&res), - "Serialize VarDesc Error. This could be a bug of Paddle."); - return res; - }) + .def("serialize_to_string", SerializeMessage) .def("persistable", &VarDescBind::Persistable) .def("set_persistable", &VarDescBind::SetPersistable); @@ -238,7 +218,9 @@ void BindVarDsec(py::module &m) { .value("SELECTED_ROWS", VarDesc::SELECTED_ROWS) .value("FEED_MINIBATCH", VarDesc::FEED_MINIBATCH) .value("FETCH_LIST", VarDesc::FETCH_LIST) - .value("STEP_SCOPES", VarDesc::STEP_SCOPES); + .value("STEP_SCOPES", VarDesc::STEP_SCOPES) + .value("LOD_RANK_TABLE", VarDesc::LOD_RANK_TABLE) + .value("LOD_TENSOR_ARRAY", VarDesc::LOD_TENSOR_ARRAY); } void BindOpDesc(py::module &m) { @@ -272,16 +254,7 @@ void BindOpDesc(py::module &m) { .def("check_attrs", &OpDescBind::CheckAttrs) .def("infer_shape", &OpDescBind::InferShape) .def("infer_var_type", &OpDescBind::InferVarType) - .def("serialize_to_string", [](OpDescBind &op_desc) -> py::bytes { - const OpDesc *desc = op_desc.Proto(); - PADDLE_ENFORCE(desc->IsInitialized(), - "OpDesc has not been initialized."); - std::string res; - PADDLE_ENFORCE( - desc->SerializeToString(&res), - "Serialize OpDesc Error. This could be a bug of Paddle."); - return res; - }); + .def("serialize_to_string", SerializeMessage); } } // namespace pybind diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 881df6ad3244e597f1c0f0121ccec08d50dd851e..0f906e0e470b7f95bb2103ae55330fc1831aa78f 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -21,14 +21,15 @@ limitations under the License. */ #include "paddle/framework/executor.h" #include "paddle/framework/feed_fetch_method.h" #include "paddle/framework/framework.pb.h" +#include "paddle/framework/lod_rank_table.h" #include "paddle/framework/lod_tensor.h" +#include "paddle/framework/lod_tensor_array.h" #include "paddle/framework/prune.h" #include "paddle/framework/selected_rows.h" #include "paddle/framework/tensor_array.h" #include "paddle/operators/cond_op.h" #include "paddle/operators/dynamic_recurrent_op.h" #include "paddle/operators/net_op.h" -#include "paddle/operators/recurrent_op.h" #include "paddle/platform/enforce.h" #include "paddle/platform/place.h" #include "paddle/pybind/exception.h" @@ -112,11 +113,13 @@ PYBIND11_PLUGIN(core) { .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) + .def("set", PyCPUTensorSetFromArray) #ifdef PADDLE_WITH_CUDA .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) + .def("set", PyCUDATensorSetFromArray) #endif .def("shape", [](Tensor &self) { return vectorize(self.dims()); }) .def("set_float_element", TensorSetElement) @@ -225,11 +228,17 @@ All parameter, weight, gradient are variables in Paddle. return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_lod_rank_table", + [](Variable &self) { return self.GetMutable(); }, + py::return_value_policy::reference) .def("get_selected_rows", [](Variable &self) -> SelectedRows * { return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_lod_tensor_array", + [](Variable &self) { return self.GetMutable(); }, + py::return_value_policy::reference) #ifdef PADDLE_WITH_CUDA .def("get_communicator", [](Variable &self) -> platform::Communicator * { @@ -428,25 +437,6 @@ All parameter, weight, gradient are variables in Paddle. return self.UnstackShared(source); }); - // recurrent_op - py::class_(m, "RecurrentOp") - .def_static( - "create", - [](py::bytes protobin) -> operators::RecurrentOp * { - OpDesc desc; - PADDLE_ENFORCE(desc.ParsePartialFromString(protobin), - "Cannot parse user input to OpDesc"); - PADDLE_ENFORCE(desc.IsInitialized(), - "User OpDesc is not initialized, reason %s", - desc.InitializationErrorString()); - auto rnn_op = OpRegistry::CreateOp(desc); - return static_cast(rnn_op.release()); - }) - .def("set_stepnet", [](operators::RecurrentOp &self, - const operators::NetOp &net) -> void { - self.set_stepnet(net.Clone()); - }); - py::class_(m, "DynamicRecurrentOp") .def_static("create", @@ -512,6 +502,32 @@ All parameter, weight, gradient are variables in Paddle. BindVarDsec(m); BindOpDesc(m); + py::class_(m, "LodRankTable") + .def("items", [](framework::LoDRankTable &table) { + std::vector> res; + for (auto &item : table.items()) { + res.push_back({item.index, item.length}); + } + return res; + }); + + py::class_(m, "LoDTensorArray") + .def("__getitem__", + [](LoDTensorArray &self, size_t i) { return &self.at(i); }, + py::return_value_policy::reference) + .def("__len__", [](LoDTensorArray &self) { return self.size(); }) + .def("__setitem__", + [](LoDTensorArray &self, size_t i, const LoDTensor &t) { + PADDLE_ENFORCE_LT(i, self.size()); + self[i].ShareDataWith(t); + self[i].set_lod(t.lod()); + }) + .def("append", [](LoDTensorArray &self, const LoDTensor &t) { + self.emplace_back(); + self.back().ShareDataWith(t); + self.back().set_lod(t.lod()); + }); + m.def("op_support_gpu", OpSupportGPU); #ifdef PADDLE_WITH_CUDA m.def("get_cuda_device_count", platform::GetCUDADeviceCount); diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index f278e79af60486bce400f313b80ebbe3971f869b..41fa658502d341fe9653a3e99b58498fcaeada47 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -85,7 +85,7 @@ struct CastToPyBufferImpl { } // namespace details inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { auto buffer_info = - details::CastToPyBufferImpl()( + details::CastToPyBufferImpl()( tensor); return buffer_info; } diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index a08716c5a559def54bb7b989f250b489f6a805a2..256500c56a2e05f981825b6ddb2a843f3ba71a83 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -2,170 +2,183 @@ set -xe -# Set BASE_IMAGE according to env variables -if [[ ${WITH_GPU} == "ON" ]]; then - BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04" -else - BASE_IMAGE="ubuntu:16.04" -fi - -DOCKERFILE_GPU_ENV="" -DOCKERFILE_CUDNN_DSO="" -if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then - DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" - DOCKERFILE_CUDNN_DSO="RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/lib/x86_64-linux-gnu/libcudnn.so" -fi - -mkdir -p /paddle/build -cd /paddle/build - -# build script will not fail if *.deb does not exist -rm *.deb 2>/dev/null || true -# delete previous built whl packages -rm -rf /paddle/paddle/dist 2>/dev/null || true - -cat </dev/null || true + # delete previous built whl packages + rm -rf /paddle/paddle/dist 2>/dev/null || true -if [ ${WITH_TESTING:-ON} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then -cat < /paddle/build/Dockerfile < -ENV HOME /root + cat <> /paddle/build/Dockerfile < /paddle/build/Dockerfile < + ENV HOME /root EOF -fi - -if [[ ${WITH_GPU} == "ON" ]]; then - NCCL_DEPS="apt-get install -y libnccl-dev &&" -else - NCCL_DEPS="" -fi - -cat >> /paddle/build/Dockerfile <> /paddle/build/Dockerfile <& parametersA = comDataA.parameters; vector& parametersB = comDataB.parameters; @@ -176,12 +167,10 @@ void compareGradient(ComData& comDataA, ComData& comDataB) { TEST(Trainer, create) { ComData dataA; - FLAGS_use_mkldnn = FLAGS_use_mkldnn_a; calcGradient(dataA, FLAGS_config_file_a); LOG(INFO) << "\n\nforwardBackward of Network A is finished\n\n"; ComData dataB; - FLAGS_use_mkldnn = FLAGS_use_mkldnn_b; calcGradient(dataB, FLAGS_config_file_b); LOG(INFO) << "\n\nforwardBackward of the Network B is finished\n\n"; diff --git a/paddle/utils/Excepts.h b/paddle/utils/Excepts.h index 0add66da7464293795927431daf0e90359f40b52..5c2c504f53a586f2991ccfae891991465fdb39b6 100644 --- a/paddle/utils/Excepts.h +++ b/paddle/utils/Excepts.h @@ -17,8 +17,7 @@ limitations under the License. */ #include -#if (defined(__APPLE__) || defined(__OSX__)) && !defined(__arm__) && \ - !defined(__aarch64__) +#if defined(__APPLE__) || defined(__OSX__) int fegetexcept(void); int feenableexcept(unsigned int excepts); diff --git a/paddle/utils/arch/osx/Excepts.cpp b/paddle/utils/arch/osx/Excepts.cpp index 42ecaa06d256c9d259a20c648626605d77ce0308..ac444615786fa9f89f96504a31b2289eae7bb643 100644 --- a/paddle/utils/arch/osx/Excepts.cpp +++ b/paddle/utils/arch/osx/Excepts.cpp @@ -14,9 +14,13 @@ limitations under the License. */ #include "paddle/utils/Excepts.h" -#if (defined(__APPLE__) || defined(__OSX__)) && !defined(__arm__) && \ - !defined(__aarch64__) - +#if defined(__APPLE__) || defined(__OSX__) +#if defined(__arm__) || defined(__arm64__) +// TODO(liuyiqun): implement the arm version +int fegetexcept(void) { return -1; } +int feenableexcept(unsigned int excepts) { return -1; } +int fedisableexcept(unsigned int excepts) { return -1; } +#else int fegetexcept(void) { static fenv_t fenv; return fegetenv(&fenv) ? -1 : (fenv.__control & FE_ALL_EXCEPT); @@ -49,5 +53,5 @@ int fedisableexcept(unsigned int excepts) { return (fesetenv(&fenv) ? -1 : old_excepts); } - +#endif #endif diff --git a/paddle/utils/tests/test_StringUtils.cpp b/paddle/utils/tests/test_StringUtils.cpp index fdc914d1bcc3c74e0f05ef475069abc315bdc306..248f58a7f26e26e82b55110930964cee04fb558b 100644 --- a/paddle/utils/tests/test_StringUtils.cpp +++ b/paddle/utils/tests/test_StringUtils.cpp @@ -18,6 +18,6 @@ limitations under the License. */ TEST(StringUtil, to) { ASSERT_NEAR(paddle::str::to("12.45"), 12.45, 1e-5); - ASSERT_DEATH(paddle::str::to("12.45x23"), ".*"); - ASSERT_DEATH(paddle::str::to(""), ".*"); + ASSERT_DEATH_IF_SUPPORTED(paddle::str::to("12.45x23"), ".*"); + ASSERT_DEATH_IF_SUPPORTED(paddle::str::to(""), ".*"); } diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index ebf0911d6ea0b39d51447859ae2aef485b50b0e6..2c2cc6245932d4af56a68d6399ce31f008bf3748 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -321,6 +321,19 @@ message ClipConfig { required double max = 2; } +message ROIPoolConfig { + required uint32 pooled_width = 1; + required uint32 pooled_height = 2; + required float spatial_scale = 3; + optional uint32 height = 4 [ default = 1 ]; + optional uint32 width = 5 [ default = 1 ]; +} + +message ScaleSubRegionConfig { + required ImageConfig image_conf = 1; + required float value = 2; +} + message LayerInputConfig { required string input_layer_name = 1; optional string input_parameter_name = 2; @@ -342,6 +355,8 @@ message LayerInputConfig { optional MultiBoxLossConfig multibox_loss_conf = 16; optional DetectionOutputConfig detection_output_conf = 17; optional ClipConfig clip_conf = 18; + optional ScaleSubRegionConfig scale_sub_region_conf = 19; + optional ROIPoolConfig roi_pool_conf = 20; } message LayerConfig { diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7bd6d59b0096c23bb791b9b50702130057628879..32578ad7799c0a276972ccef7770c2eae8438069 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -44,6 +44,7 @@ add_custom_target(copy_paddle_pybind ALL DEPENDS ${PADDLE_SOURCE_DIR}/python/pad add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp + COMMAND touch stub.cc COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 32ef7ca4860598ff4dc095b523fdbe33801e62dc..d3923387ce27f7f37d6eaec49f5fe26d9e03fdc4 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1987,6 +1987,18 @@ class DetectionOutputLayer(LayerBase): self.config.size = size +@config_layer('roi_pool') +class ROIPoolLayer(LayerBase): + def __init__(self, name, inputs, pooled_width, pooled_height, spatial_scale, + num_channels, **xargs): + super(ROIPoolLayer, self).__init__(name, 'roi_pool', 0, inputs) + config_assert(len(inputs) == 2, 'ROIPoolLayer must have 2 inputs') + self.config.inputs[0].roi_pool_conf.pooled_width = pooled_width + self.config.inputs[0].roi_pool_conf.pooled_height = pooled_height + self.config.inputs[0].roi_pool_conf.spatial_scale = spatial_scale + self.set_cnn_layer(name, pooled_height, pooled_width, num_channels) + + @config_layer('data') class DataLayer(LayerBase): def __init__(self, @@ -2793,9 +2805,15 @@ class NCELayer(LayerBase): @config_layer('addto') class AddToLayer(LayerBase): + layer_type = 'addto' + def __init__(self, name, inputs, bias=True, **xargs): + use_mkldnn = bool(int(g_command_config_args.get("use_mkldnn", 0))) + if self.layer_type == "mkldnn_addto": + config_assert(use_mkldnn, "mkldnn_addto only support MKLDNN") + self.layer_type = 'mkldnn_addto' if use_mkldnn else 'addto' super(AddToLayer, self).__init__( - name, 'addto', 0, inputs=inputs, **xargs) + name, self.layer_type, 0, inputs=inputs, **xargs) config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') if len(self.inputs) > 1: @@ -2814,6 +2832,11 @@ class AddToLayer(LayerBase): self.create_bias_parameter(bias, self.config.size) +@config_layer('mkldnn_addto') +class MKLDNNAddtoLayer(AddToLayer): + layer_type = 'mkldnn_addto' + + @config_layer('agent') class AgentLayer(LayerBase): def __init__(self, name, size, device=None): @@ -3808,6 +3831,25 @@ class SwitchOrderLayer(LayerBase): self.config.reshape_conf.width_axis.extend(reshape['width']) +@config_layer('scale_sub_region') +class ScaleSubRegionLayer(LayerBase): + def __init__(self, name, inputs, value, **xargs): + super(ScaleSubRegionLayer, self).__init__( + name, 'scale_sub_region', 0, inputs=inputs, **xargs) + scale_sub_region_conf = self.config.inputs[0].scale_sub_region_conf + scale_sub_region_conf.value = value + + # get channel, width and height from input_0 layer + input_layer = self.get_input_layer(0) + image_conf = scale_sub_region_conf.image_conf + image_conf.img_size = input_layer.width + image_conf.img_size_y = input_layer.height + image_conf.channels = input_layer.size / (input_layer.width * + input_layer.height) + self.set_cnn_layer(name, image_conf.img_size_y, image_conf.img_size, + image_conf.channels) + + # Deprecated, use a new layer specific class instead @config_func def Layer(name, type, **xargs): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 00b35f4372ffd01c19b1bffa8c0e9e347eac159e..8be957232e14adda7fc8b82e9e8e9ae8fcc53560 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -122,6 +122,7 @@ __all__ = [ 'cross_channel_norm_layer', 'multibox_loss_layer', 'detection_output_layer', + 'roi_pool_layer', 'spp_layer', 'pad_layer', 'eos_layer', @@ -143,6 +144,8 @@ __all__ = [ 'scale_shift_layer', 'img_conv3d_layer', 'resize_layer', + 'sub_seq_layer', + 'scale_sub_region_layer', ] @@ -219,6 +222,7 @@ class LayerType(object): PRIORBOX_LAYER = 'priorbox' MULTIBOX_LOSS_LAYER = 'multibox_loss' DETECTION_OUTPUT_LAYER = 'detection_output' + ROI_POOL_LAYER = 'roi_pool' CTC_LAYER = 'ctc' WARP_CTC_LAYER = 'warp_ctc' @@ -252,6 +256,9 @@ class LayerType(object): SCALE_SHIFT_LAYER = 'scale_shift' RESIZE = 'resize' + SUB_SEQ_LAYER = 'subseq' + + SCALE_SUB_REGION_LAYER = 'scale_sub_region' @staticmethod def is_layer_type(type_name): @@ -784,10 +791,9 @@ class MixedLayerType(LayerOutput): :type size: int :param act: Activation type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None @@ -884,10 +890,9 @@ def mixed_layer(size=0, then this function will just return layer's name. :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: The extra layer config. Default is None. :type layer_attr: ExtraLayerAttribute @@ -1029,10 +1034,9 @@ def fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute|list. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -1047,6 +1051,13 @@ def fc_layer(input, if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: + if "parameter_name" in param_attr.attr and len(input) > 1: + logger.fatal( + "When the name field of param_attr is manually specified " + "and the input is a list, the param_attr should also be a " + "list with each item being the param_attr for each input " + "item. If only one named param_attr is provided, all the " + "input items would share this parameter.") param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] assert isinstance(input, collections.Sequence) @@ -1296,6 +1307,50 @@ def detection_output_layer(input_loc, name, LayerType.DETECTION_OUTPUT_LAYER, parents=parents, size=size) +@wrap_name_default("roi_pool") +def roi_pool_layer(input, + rois, + pooled_width, + pooled_height, + spatial_scale, + num_channels=None, + name=None): + """ + A layer used by Fast R-CNN to extract feature maps of ROIs from the last + feature map. + + :param name: The Layer Name. + :type name: basestring + :param input: The input layer. + :type input: LayerOutput. + :param rois: The input ROIs' data. + :type rois: LayerOutput. + :param pooled_width: The width after pooling. + :type pooled_width: int + :param pooled_height: The height after pooling. + :type pooled_height: int + :param spatial_scale: The spatial scale between the image and feature map. + :type spatial_scale: float + :param num_channels: number of input channel. + :type num_channels: int + :return: LayerOutput + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + size = num_channels * pooled_width * pooled_height + Layer( + name=name, + type=LayerType.ROI_POOL_LAYER, + inputs=[input.name, rois.name], + pooled_width=pooled_width, + pooled_height=pooled_height, + spatial_scale=spatial_scale, + num_channels=num_channels) + return LayerOutput( + name, LayerType.ROI_POOL_LAYER, parents=[input, rois], size=size) + + @wrap_name_default("cross_channel_norm") def cross_channel_norm_layer(input, name=None, param_attr=None): """ @@ -1378,10 +1433,9 @@ def pooling_layer(input, :type pooling_type: BasePoolingType | None :param stride: The step size between successive pooling regions. :type stride: Int - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: The Extra Attributes for layer, such as dropout. :type layer_attr: ExtraLayerAttribute | None @@ -1479,10 +1533,9 @@ def lstmemory(input, :type gate_act: BaseActivation :param state_act: state activation type, TanhActivation by default. :type state_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute | None | False @@ -1605,10 +1658,9 @@ def grumemory(input, This activation affects the :math:`z_t` and :math:`r_t`. It is the :math:`\\sigma` in the above formula. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute | None | False @@ -1805,10 +1857,9 @@ def expand_layer(input, :type expand_as: LayerOutput :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param expand_level: whether input layer is timestep(default) or sequence. :type expand_level: ExpandLevel @@ -1927,10 +1978,9 @@ def seq_reshape_layer(input, :type act: BaseActivation :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -2314,10 +2364,9 @@ def hsigmoid(input, :type num_classes: int | None :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: Parameter Attribute. None means default parameter. :type param_attr: ParameterAttribute | None @@ -2457,10 +2506,9 @@ def img_conv_layer(input, :type dilation: int | tuple | list :param dilation_y: The y dimension of the dilation. :type dilation_y: int - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param num_channels: number of input channels. If None will be set automatically from previous output. @@ -3209,10 +3257,9 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): :type input: LayerOutput | list | tuple :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute @@ -3365,10 +3412,9 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, :type act: BaseActivation :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -3548,10 +3594,9 @@ def lstm_step_layer(input, :type gate_act: BaseActivation :param state_act: State Activation Type. TanhActivation is the default. :type state_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute @@ -3607,10 +3652,9 @@ def gru_step_layer(input, :param name: The name of this layer. It is optional. :param gate_act: Activation type of this layer's two gates. Default is Sigmoid. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: the parameter_attribute for transforming the output_mem from previous step. @@ -3670,10 +3714,9 @@ def gru_step_naive_layer(input, :type act: BaseActivation :param gate_act: Activation type of this layer's two gates. Default is Sigmoid. :type gate_act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: :param layer_attr: @@ -3803,10 +3846,9 @@ def recurrent_layer(input, :type input: LayerOutput :param act: Activation type. TanhActivation is the default. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param param_attr: parameter attribute. :type param_attr: ParameterAttribute @@ -4796,10 +4838,9 @@ def tensor_layer(a, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -4861,10 +4902,9 @@ def selective_fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute | None @@ -4879,6 +4919,13 @@ def selective_fc_layer(input, if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: + if "parameter_name" in param_attr.attr and len(input) > 1: + logger.fatal( + "When the name field of param_attr is manually specified " + "and the input is a list, the param_attr should also be a " + "list with each item being the param_attr for each input " + "item. If only one named param_attr is provided, all the " + "input items would share this parameter.") param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] assert isinstance(input, collections.Sequence) @@ -5480,7 +5527,11 @@ def crf_decoding_layer(input, return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) -@wrap_act_default(act=SigmoidActivation()) +""" +Following are cost Layers. +""" + + @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @wrap_name_default() @@ -5488,7 +5539,6 @@ def crf_decoding_layer(input, def nce_layer(input, label, num_classes=None, - act=None, param_attr=None, weight=None, num_neg_samples=10, @@ -5497,9 +5547,12 @@ def nce_layer(input, bias_attr=None, layer_attr=None): """ - Noise-contrastive estimation. - Implements the method in the following paper: - A fast and simple algorithm for training neural probabilistic language models. + Noise-contrastive estimation. This layer implements the method in the + following paper: + + Reference: + A fast and simple algorithm for training neural probabilistic language + models. https://www.cs.toronto.edu/~amnih/papers/ncelm.pdf The example usage is: @@ -5511,32 +5564,37 @@ def nce_layer(input, :param name: The name of this layer. It is optional. :type name: basestring - :param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :param input: The input layers. It should be a LayerOutput or a list/tuple + of LayerOutput. :type input: LayerOutput | list | tuple | collections.Sequence - :param label: label layer + :param label: The ground truth. :type label: LayerOutput - :param weight: weight layer, can be None(default) + :param weight: The weight layer defines a weight for each sample in the + mini-batch. The default value is None. :type weight: LayerOutput - :param num_classes: number of classes. + :param num_classes: The class number. :type num_classes: int - :param act: Activation type. SigmoidActivation is the default. - :type act: BaseActivation - :param param_attr: The Parameter Attribute|list. - :type param_attr: ParameterAttribute - :param num_neg_samples: number of negative samples. Default is 10. + :param param_attr: The parameter attributes. + :type param_attr: ParameterAttribute|list + :param num_neg_samples: The number of sampled negative labels. The default + value is 10. :type num_neg_samples: int - :param neg_distribution: The distribution for generating the random negative labels. - A uniform distribution will be used if not provided. - If not None, its length must be equal to num_classes. + :param neg_distribution: The discrete noisy distribution over the output + space from which num_neg_samples negative labels + are sampled. If this parameter is not set, a + uniform distribution will be used. A user defined + distribution is a list whose length must be equal + to the num_classes. Each member of the list defines + the probability of a class given input x. :type neg_distribution: list | tuple | collections.Sequence | None - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The attribute for bias. If this parameter is set False or + any object whose type is not ParameterAttribute, no bias + is added. If this parameter is set True, the bias is + initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :return: layer name. + :return: The LayerOutput object. :rtype: LayerOutput """ if isinstance(input, LayerOutput): @@ -5559,8 +5617,6 @@ def nce_layer(input, assert isinstance(neg_distribution, collections.Sequence) assert len(neg_distribution) == num_classes assert abs(sum(neg_distribution) - 1.0) < 1e-5 - if not isinstance(act, BaseActivation): - raise TypeError() ipts_for_layer = [] parents = [] @@ -5582,7 +5638,7 @@ def nce_layer(input, type=LayerType.NCE_LAYER, num_classes=num_classes, neg_sampling_dist=neg_distribution, - active_type=act.name, + active_type=SigmoidActivation().name, num_neg_samples=num_neg_samples, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), @@ -5592,12 +5648,7 @@ def nce_layer(input, LayerType.NCE_LAYER, parents=parents, size=l.config.size, - activation=act) - - -""" -following are cost Layers. -""" + activation=SigmoidActivation()) @wrap_name_default() @@ -5756,20 +5807,21 @@ def cross_entropy(input, :param input: The first input layer. :type input: LayerOutput. :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The cost is multiplied with coeff. - The coefficient affects the gradient in the backward. - :type coeff: float. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float :param weight: The cost of each sample is multiplied with each weight. The weight should be a layer with size=1. Note that gradient will not be calculated for weight. :type weight: LayerOutout - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ ipts, parents = __cost_input__(input, label, weight) @@ -5802,19 +5854,21 @@ def cross_entropy_with_selfnorm(input, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float :param softmax_selfnorm_alpha: The scale factor affects the cost. - :type softmax_selfnorm_alpha: float. - :param layer_attr: Extra Layer Attribute. + :type softmax_selfnorm_alpha: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ Layer( name=name, @@ -5835,7 +5889,7 @@ def cross_entropy_with_selfnorm(input, @layer_support() def sum_cost(input, name=None, layer_attr=None): """ - A loss layer which calculate the sum of the input as loss + A loss layer which calculates the sum of the input as loss. The example usage is: @@ -5844,10 +5898,11 @@ def sum_cost(input, name=None, layer_attr=None): cost = sum_cost(input=input_layer) :param input: The input of this layer. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param layer_attr: Extra Layer Attribute. + :type name: basestring + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput. @@ -5887,16 +5942,18 @@ def huber_regression_cost(input, cost = huber_regression_cost(input=input_layer, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. + :type name: basestring :param delta: The difference between the observed and predicted values. - :type delta: float. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. - :param layer_attr: Extra Layer Attribute. + :type delta: float + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput. @@ -5937,17 +5994,19 @@ def huber_classification_cost(input, cost = huber_classification_cost(input=input_layer, label=label_layer) :param input: The first input layer. - :type input: LayerOutput. + :type input: LayerOutput :param label: The input label. - :type input: LayerOutput. + :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring. - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float. - :param layer_attr: Extra Layer Attribute. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. + :type coeff: float + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. - :rtype: LayerOutput. + :rtype: LayerOutput """ assert isinstance(input, LayerOutput) if input.size is not None: @@ -5984,10 +6043,12 @@ def multi_binary_label_cross_entropy(input, :param label: The input label. :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring - :param coeff: The coefficient affects the gradient in the backward. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. :type coeff: float - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput @@ -6090,7 +6151,7 @@ def cross_entropy_over_beam(input, name=None): :param input: Input beams for this layer. :type input: BeamInput - :param name: The name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6125,7 +6186,7 @@ def cross_entropy_over_beam(input, name=None): def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): """ This is a L1 loss but more smooth. It requires that the - size of input and label are equal. The formula is as follows, + sizes of input and label are equal. The formula is as follows, .. math:: @@ -6137,8 +6198,9 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): smooth_{L1}(x) = \\begin{cases} 0.5x^2& \\text{if} \\ |x| < 1 \\\\ |x|-0.5& \\text{otherwise} \end{cases} - More details can be found by referring to `Fast R-CNN - `_ + Reference: + Fast R-CNN + https://arxiv.org/pdf/1504.08083v2.pdf The example usage is: @@ -6152,10 +6214,12 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): :param label: The input label. :type input: LayerOutput :param name: The name of this layer. It is optional. - :type name: None | basestring - :param coeff: The coefficient affects the gradient in the backward. + :type name: basestring + :param coeff: The weight of the gradient in the back propagation. + 1.0 is the default. :type coeff: float - :param layer_attr: Extra Layer Attribute. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput @@ -6177,12 +6241,12 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): @wrap_name_default() def multiplex_layer(input, name=None, layer_attr=None): """ - This layer multiplex multiple layers according to the index, - which is provided by the first input layer. - inputs[0]: the index of the layer to output of size batchSize. + This layer multiplex multiple layers according to the indexes, + which are provided by the first input layer. + inputs[0]: the indexes of the layers to form the output of size batchSize. inputs[1:N]; the candidate output data. - For each index i from 0 to batchSize -1, the output is the i-th row of the - (index[i] + 1)-th layer. + For each index i from 0 to batchSize - 1, the i-th row of the output is the + the same to the i-th row of the (index[i] + 1)-th layer. For each i-th row of output: .. math:: @@ -6201,7 +6265,8 @@ def multiplex_layer(input, name=None, layer_attr=None): :type input: list of LayerOutput :param name: The name of this layer. It is optional. :type name: basestring - :param layer_attr: extra layer attributes. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute. :return: LayerOutput object. :rtype: LayerOutput @@ -6305,14 +6370,14 @@ def row_conv_layer(input, :type context_len: int :param act: Activation Type. LinearActivation is the default. :type act: BaseActivation - :param param_attr: The Parameter Attribute. If None, the parameter will be - initialized smartly. It's better to set it by yourself. + :param param_attr: The parameter attribute. See ParameterAttribute for + details. :type param_attr: ParameterAttribute - :param layer_attr: Extra Layer config. + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput - """ assert isinstance(input, LayerOutput) assert context_len > 0, "the context_len must be greatet than 0." @@ -6337,7 +6402,7 @@ def prelu_layer(input, param_attr=None, layer_attr=None): """ - The Parameter Relu activation that actives outputs with a learnable weight. + The Parametric Relu activation that actives outputs with a learnable weight. Reference: Delving Deep into Rectifiers: Surpassing Human-Level Performance on @@ -6357,16 +6422,17 @@ def prelu_layer(input, :type name: basestring :param input: The input of this layer. :type input: LayerOutput - :param partial_sum: this parameter makes a group of inputs share a same weight. + :param partial_sum: this parameter makes a group of inputs share the same weight. - partial_sum = 1, indicates the element-wise activation: each element has a weight. - - partial_sum = number of elements in one channel, indicates the channel-wise activation, elements in a channel share a same weight. - - partial_sum = number of outputs, indicates all elements share a same weight. + - partial_sum = number of elements in one channel, indicates the channel-wise activation, elements in a channel share the same weight. + - partial_sum = number of outputs, indicates all elements share the same weight. :type partial_sum: int :param param_attr: The parameter attribute. See ParameterAttribute for details. - :type param_attr: ParameterAttribute | None - :param layer_attr: Extra layer configurations. Default is None. + :type param_attr: ParameterAttribute + :param layer_attr: The extra layer attribute. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -6422,34 +6488,34 @@ def gated_unit_layer(input, :param input: The input of this layer. :type input: LayerOutput - :param size: output size of the gated unit. + :param size: The dimension of this layer's output. :type size: int - :param act: Activation type of the projected input. LinearActivation is the default. + :param act: Activation type of the projection. LinearActivation is the default. :type act: BaseActivation :param name: The name of this layer. It is optional. :type name: basestring - :param gate_attr: Attributes to tune the gate output, for example, error - clipping threshold, dropout and so on. See ExtraLayerAttribute for - more details. + :param gate_attr: The extra layer attribute of the gate. See ExtraLayerAttribute for + details. :type gate_attr: ExtraLayerAttribute | None - :param gate_param_attr: Attributes to tune the learnable projected matrix - parameter of the gate. - :type gate_param_attr: ParameterAttribute | None - :param gate_bias_attr: Attributes to tune the learnable bias of the gate. - :type gate_bias_attr: ParameterAttribute | None - :param inproj_attr: Attributes to the tune the projected input, for - example, error clipping threshold, dropout and so on. See - ExtraLayerAttribute for more details. + :param gate_param_attr: The parameter attribute of the gate. See ParameterAttribute + for details. + :type gate_param_attr: ParameterAttribute + :param gate_bias_attr: The bias attribute of the gate. If the parameter is set to False or + an object whose type is not ParameterAttribute, no bias is defined. + If the parameter is set to True, the bias is initialized to zero. + :type gate_bias_attr: ParameterAttribute | bool | None | Any + :param inproj_attr: Extra layer attributes of the projection. See ExtraLayerAttribute for + details. :type inproj_attr: ExtraLayerAttribute | None - :param inproj_param_attr: Attributes to tune the learnable parameter of - the projection of input. - :type inproj_param_attr: ParameterAttribute | None - :param inproj_bias_attr: Attributes to tune the learnable bias of - projection of the input. - :type inproj_bias_attr: ParameterAttribute | None - :param layer_attr: Attributes to tune the final output of the gated unit, - for example, error clipping threshold, dropout and so on. See - ExtraLayerAttribute for more details. + :param inproj_param_attr: The parameter attribute of the projection. See ParameterAttribute + for details. + :type inproj_param_attr: ParameterAttribute + :param inproj_bias_attr: The bias attribute of the projection. If the parameter is set to False + or an object whose type is not ParameterAttribute, no bias is defined. + If the parameter is set to True, the bias is initialized to zero. + :type inproj_bias_attr: ParameterAttribute | bool | None | Any + :param layer_attr: Extra layer attribute of the product. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute | None :return: LayerOutput object. :rtype: LayerOutput @@ -6534,26 +6600,27 @@ def switch_order_layer(input, @layer_support() def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None): """ - This layer crops images by offset and shape. User can set crop shape by - args 'shape' explicitly or by reference input layer. + This layer crops images according to the offset and shape. Users can set + the crop shape through the argument 'shape' explicitly or by specifying a + reference input layer. The example usage is: .. code-block:: python crop = crop_layer(input=[image_input, reference_input], axis=2, offset=[2, 3]) - :param input: The input of this layer. If two inputs are given, the second input - will be regarded as reference input. + :param input: The input of this layer. If two inputs are given, the second one + will be regarded as the reference. :type input: LayerOutput | Sequence :param offset: The crop offset. :type offset: Sequence - :param axis: start axis to be cropped. To image input layer: + :param axis: The start axis to be cropped. For image input layer: - 0: batch size - 1: channels - 2: height - 3: width - :type partial_sum: int - :param shape: The shape to be cropped. Default is None. + :type axis: int + :param shape: The shape to be cropped to. Default is None. :type shape: Sequence | None :param name: The name of this layer. It is optional. :type name: basestring @@ -6644,9 +6711,9 @@ def clip_layer(input, min, max, name=None): :param input: The input of this layer. :type input: LayerOutput. :param min: The lower threshold for clipping. - :type min: double + :type min: float :param max: The upper threshold for clipping. - :type max: double + :type max: float :return: LayerOutput object. :rtype: LayerOutput """ @@ -6688,13 +6755,12 @@ def seq_slice_layer(input, starts, ends, name=None): :type name: basestring :param input: The input of this layer, which should be a sequence. :type input: LayerOutput - :param starts: start indices to slice the input sequence. + :param starts: The start indices to slice the input sequence. :type starts: LayerOutput | None - :param ends: end indices to slice the input sequence. + :param ends: The end indices to slice the input sequence. :type ends: LayerOutput | None :return: LayerOutput object. :rtype: LayerOutput - """ assert isinstance(input, LayerOutput), ( @@ -6730,7 +6796,7 @@ def seq_slice_layer(input, starts, ends, name=None): @layer_support() def kmax_seq_score_layer(input, name=None, beam_size=1): """ - This layer accepts one input which are scores over a sequence or a nested + This layer accepts one input which is scores over a sequence or a nested sequence, and returns indices of beam_size sequences with highest scores. .. code-block:: python @@ -6740,11 +6806,11 @@ def kmax_seq_score_layer(input, name=None, beam_size=1): :param name: The name of this layer. It is optional. :type name: basestring - :param input: The input of this layer. It stores scores over a sequence or a nested - sequence and its size must be 1. + :param input: The input of this layer. It stores scores over a sequence or + a nested sequence and its size must be 1. :type input: LayerOutput - :param beam_size: sequence indices with top beam_size scores are returned. - :type beam_size: double + :param beam_size: The indices of the sequences with top beam_size scores are returned. + :type beam_size: int :return: LayerOutput object. :rtype: LayerOutput """ @@ -6800,38 +6866,43 @@ def img_conv3d_layer(input, :type name: basestring :param input: The input of this layer. :type input: LayerOutput - :param filter_size: The x dimension of a filter kernel. Or input a list. + :param filter_size: The dimensions of the filter kernel along three axises. If the parameter + is set to one integer, the three dimensions will be same. :type filter_size: int | tuple | list - :param num_filters: Each filter group's number of filter + :param num_filters: The number of filters in each group. + :type num_filters: int :param act: Activation type. ReluActivation is the default. :type act: BaseActivation - :param groups: Group size of filters. + :param groups: The number of the filter groups. :type groups: int - :param stride: The x dimension of the stride. Or input a tuple for two image - dimension. + :param stride: The strides of the convolution along three axises. If the parameter + is set to one integer, the three strides will be same. :type stride: int | tuple | list - :param padding: The x dimension of the padding. Or input a tuple for two - image dimension + :param padding: The numbers of padding along three axises. If the parameter is set to + one integer, they will be same. :type padding: int | tuple | list - :param bias_attr: Convolution bias attribute. None means default bias. - False means no bias. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any - :param num_channels: number of input channels. If None will be set - automatically from previous output. + :param num_channels: The number of input channels. If the parameter is not set or + set to None, its actual value will be automatically set to + the channels number of the input . :type num_channels: int - :param param_attr: Convolution param attribute. None means default attribute + :param param_attr: The parameter attribute of the convolution. See ParameterAttribute for + details. :type param_attr: ParameterAttribute - :param shared_biases: Is biases will be shared between filters or not. + :param shared_biases: Whether biases will be shared between filters or not. :type shared_biases: bool - :param layer_attr: Layer Extra Attribute. + :param layer_attr: The extra layer attributes. See ExtraLayerAttribute for + details. :type layer_attr: ExtraLayerAttribute - :param trans: true if it is a convTransLayer, false if it is a convLayer + :param trans: True if it is a convTransLayer, False if it is a convLayer :type trans: bool - :param layer_type: specify the layer_type, default is None. If trans=True, - layer_type has to be "exconvt" or "cudnn_convt", - otherwise layer_type has to be either "exconv" or - "cudnn_conv" - :type layer_type: String + :param layer_type: Specify the layer_type. If the parameter is set, it must be "deconv3d" + when trans=True. If not set, it will be automatically set to "deconv3d" + when trans=True and "conv3d" when trans=False. + :type layer_type: basestring :return: LayerOutput object. :rtype: LayerOutput """ @@ -6913,7 +6984,7 @@ def img_conv3d_layer(input, def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): """ A layer applies a linear transformation to each element in each row of - the input matrix. For each element, the layer first re-scale it and then + the input matrix. For each element, the layer first re-scales it and then adds a bias to it. This layer is very like the SlopeInterceptLayer, except the scale and @@ -6931,12 +7002,12 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): :type name: basestring :param input: The input of this layer. :type input: LayerOutput - :param param_attr: The parameter attribute of scaling. + :param param_attr: The parameter attribute of scaling. See ParameterAttribute for + details. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If the parameter is set to - False or something not type of ParameterAttribute, - no bias is defined. If the parameter is set to - True, the bias is initialized to zero. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. :type bias_attr: ParameterAttribute | None | bool | Any :return: LayerOutput object. :rtype: LayerOutput @@ -6968,3 +7039,108 @@ def resize_layer(input, size, name=None): """ Layer(name=name, type=LayerType.RESIZE, inputs=Input(input.name), size=size) return LayerOutput(name, LayerType.RESIZE, parents=[input], size=input.size) + + +@wrap_act_default(act=LinearActivation()) +@wrap_name_default('sub_seq') +def sub_seq_layer(input, offsets, sizes, act=None, bias_attr=None, name=None): + """ + sub_seq_layer will return sub-sequences from the input sequences. For each + sequence in the input sequence layer, sub_seq_layer will slice it by given + offset and size. Please notice that, number of offset value and size value + both are equal to the number of sequence in the input layer. + + .. code-block:: python + + sub_seq = sub_seq_layer(input=input_seq, offsets=offsets, sizes=sizes) + + :param name: The name of this layer. It is optional. + :type name: basestring + :param input: The input of this layer, which should be sequence. + :type input: LayerOutput + :param offsets: The offset indices to slice the input sequence, which should + be sequence type. + :type offsets: LayerOutput + :param sizes: The sizes of the sub-sequences, which should be sequence type. + :type sizes: LayerOutput + :param act: Activation type, LinearActivation is the default. + :type act: BaseActivation. + :param bias_attr: The bias attribute. If the parameter is set to False or an object + whose type is not ParameterAttribute, no bias is defined. If the + parameter is set to True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute | None | bool | Any + :return: LayerOutput object. + :rtype: LayerOutput + """ + + assert isinstance(input, LayerOutput), ( + 'The first input of sub_seq_layer layer must be a PaddlePaddle layer.') + assert isinstance(offsets, LayerOutput), ( + 'The offset indices for sub_seq_layer, ' + 'must be a PaddlePaddle layer.') + assert isinstance(sizes, LayerOutput), ( + 'The sizes of sub-sequences, must be a PaddlePaddle layer.') + + Layer( + name=name, + type=LayerType.SUB_SEQ_LAYER, + inputs=[input.name, offsets.name, sizes.name], + active_type=act.name, + bias=ParamAttr.to_bias(bias_attr)) + + return LayerOutput( + name, + LayerType.SUB_SEQ_LAYER, + parents=[input, offsets, sizes], + size=input.size) + + +@wrap_name_default('scale_sub_region') +def scale_sub_region_layer(input, indices, value, name=None): + """ + Given an image or feature map with CHW information, scale_sub_region_layer + can be used to multiply a real value to values of a sub continuous region. + You can provide start and end indices of CHW for each instance. + Please notice that all start indices are counting from 1. + The shape of indices should be [batch_size, 6] and the layout for each row + is [C_Start, C_End, H_Start, H_End, W_Start, W_End]. + + .. code-block:: python + + scale_sub_region = scale_sub_region_layer(input=input, + indices=indices, + value=value) + + :param name: The name of this layer. It is optional. + :type name: basestring + :param input: The input of this layer which should contains CHW information. + :type input: LayerOutput + :param indices: Start index and end index for C H W, the input value should + be a 2-D matrix with shape [batch_size, 6]. + :type indices: LayerOutput. + :param value: value to multiply. + :type value: float + :return: LayerOutput object. + :rtype: LayerOutput + """ + + assert isinstance(input, LayerOutput), ( + 'The first input of scale_sub_region_layer, ' + 'must be a PaddlePaddle layer.') + assert isinstance(indices, LayerOutput), ( + 'The start and end indices for CHW, must be a PaddlePaddle layer.') + assert isinstance(value, float), ( + 'The value to multiply, must be a real value.') + + Layer( + name=name, + type=LayerType.SCALE_SUB_REGION_LAYER, + inputs=[input.name, indices.name], + value=value) + + return LayerOutput( + name, + LayerType.SCALE_SUB_REGION_LAYER, + parents=[input, indices], + num_filters=input.num_filters, + size=input.size) diff --git a/python/paddle/trainer_config_helpers/optimizers.py b/python/paddle/trainer_config_helpers/optimizers.py index c3495ee110bfaf91a47637a52e88b3bb56dce7a9..c3cd4cf8c32e20f3ef86305489fc415397dec1b8 100644 --- a/python/paddle/trainer_config_helpers/optimizers.py +++ b/python/paddle/trainer_config_helpers/optimizers.py @@ -116,7 +116,7 @@ class AdamOptimizer(BaseSGDOptimizer): m(w, t) & = \\beta_1 m(w, t-1) + (1 - \\beta_1) \\nabla Q_i(w) \\\\ v(w, t) & = \\beta_2 v(w, t-1) + (1 - \\beta_2)(\\nabla Q_i(w)) ^2 \\\\ - w & = w - \\frac{\\eta}{\\sqrt{v(w,t) + \\epsilon}} + w & = w - \\frac{\\eta m(w, t)}{\\sqrt{v(w,t) + \\epsilon}} :param beta1: the :math:`\\beta_1` in equation. :type beta1: float diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index 6a4550c209762362d40f8a2afaf526a1fe53ca6b..1c7451e0abf5dc1b99671f292e2ffc2d2282abe9 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -9,7 +9,7 @@ test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer test_recursive_topology test_gated_unit_layer test_clip_layer test_row_l2_norm_layer test_kmax_seq_socre_layer test_sub_nested_seq_select_layer test_scale_shift_layer -test_seq_slice_layer test_cross_entropy_over_beam test_pooling3D_layer -test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer) +test_seq_slice_layer test_cross_entropy_over_beam test_roi_pool_layer test_pooling3D_layer +test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer test_scale_sub_region_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr new file mode 100644 index 0000000000000000000000000000000000000000..f1bc65b3aee7488700a9d24e049adb510649c475 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_roi_pool_layer.protostr @@ -0,0 +1,98 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 588 + active_type: "" + height: 14 + width: 14 +} +layers { + name: "rois" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 3136 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 3 + channels: 3 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 3 + output_x: 14 + img_size: 14 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + output_y: 14 + img_size_y: 14 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 16 + shared_biases: true + height: 14 + width: 14 +} +layers { + name: "__roi_pool_0__" + type: "roi_pool" + size: 784 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + roi_pool_conf { + pooled_width: 7 + pooled_height: 7 + spatial_scale: 0.0625 + } + } + inputs { + input_layer_name: "rois" + } + height: 7 + width: 7 +} +parameters { + name: "___conv_0__.w0" + size: 432 + initial_mean: 0.0 + initial_std: 0.272165526976 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 16 + initial_mean: 0.0 + initial_std: 0.0 + dims: 16 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "rois" +output_layer_names: "__roi_pool_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "rois" + layer_names: "__conv_0__" + layer_names: "__roi_pool_0__" + input_layer_names: "data" + input_layer_names: "rois" + output_layer_names: "__roi_pool_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr new file mode 100644 index 0000000000000000000000000000000000000000..d20133a10ec605654bd3744297673068a77020b8 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_sub_region_layer.protostr @@ -0,0 +1,51 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 2016 + active_type: "" + height: 48 + width: 42 +} +layers { + name: "indices" + type: "data" + size: 6 + active_type: "" +} +layers { + name: "__scale_sub_region_0__" + type: "scale_sub_region" + size: 2016 + active_type: "" + inputs { + input_layer_name: "data" + scale_sub_region_conf { + image_conf { + channels: 1 + img_size: 42 + img_size_y: 48 + } + value: 0.0 + } + } + inputs { + input_layer_name: "indices" + } + height: 48 + width: 42 +} +input_layer_names: "data" +input_layer_names: "indices" +output_layer_names: "__scale_sub_region_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "indices" + layer_names: "__scale_sub_region_0__" + input_layer_names: "data" + input_layer_names: "indices" + output_layer_names: "__scale_sub_region_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..b739a81b8505c94a2312ac735647fb114982f1f7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_roi_pool_layer.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +data = data_layer(name='data', size=3 * 14 * 14, height=14, width=14) + +rois = data_layer(name='rois', size=10) + +conv = img_conv_layer( + input=data, + filter_size=3, + num_channels=3, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +roi_pool = roi_pool_layer( + input=conv, + rois=rois, + pooled_width=7, + pooled_height=7, + spatial_scale=1. / 16) + +outputs(roi_pool) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..8d4bf28bf1eaf58e1fd0eb62fd10efe998587edd --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_sub_region_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +data = data_layer(name='data', size=2016, height=48, width=42) +indices = data_layer(name='indices', size=6) + +scale_sub_region = scale_sub_region_layer( + input=data, indices=indices, value=0.0) + +outputs(scale_sub_region) diff --git a/python/paddle/utils/merge_model.py b/python/paddle/utils/merge_model.py index 48e5087cc281bd3a3d0b4a403372456ebbf39c62..421e953d2775f145800cf7179ec644697a265060 100644 --- a/python/paddle/utils/merge_model.py +++ b/python/paddle/utils/merge_model.py @@ -23,32 +23,32 @@ from paddle.v2.topology import Topology def merge_v2_model(net, param_file, output_file): - '''Integrate the model config and model parameters into one file. - + '''Merge the model config and parameters into one file. + The model configuration file describes the model structure which ends with .py. The parameters file stores the parameters of the model which ends with .tar.gz. - - @param net The output layer of the network. - @param param_file Path of the model parameters(.tar.gz) which is stored by v2 api. + + @param net The output layer of the network for inference. + @param param_file Path of the parameters (.tar.gz) which is stored by v2 api. @param output_file Path of the merged file which will be generated. - + Usage: - from paddle.util.merge_model import merge_v2_model + from paddle.utils.merge_model import merge_v2_model # import your network configuration - from mobilenet import mobile_net - - net = mobile_net(3*224*224, 102) + from example_net import net_conf + + net = net_conf(is_predict=True) param_file = './param_pass_00000.tar.gz' output_file = './output.paddle' - + merge_v2_model(net, param_file, output_file) ''' assert isinstance(net, LayerOutput), \ - "The net should be the output of the network" + "The net should be the output of the network for inference" assert os.path.exists(param_file), \ "The model parameters file %s does not exists " % (param_file) diff --git a/python/paddle/v2/dataset/uci_housing.py b/python/paddle/v2/dataset/uci_housing.py index ce60aa21c2ad1fb8f089d19d548b59a8c806d1ee..98b97c75ca72f11c105535e0f2a5fa0201db5d42 100644 --- a/python/paddle/v2/dataset/uci_housing.py +++ b/python/paddle/v2/dataset/uci_housing.py @@ -22,6 +22,7 @@ parse training set and test set into paddle reader creators. import numpy as np import os import paddle.v2.dataset.common +from paddle.v2.parameters import Parameters __all__ = ['train', 'test'] @@ -34,7 +35,8 @@ feature_names = [ UCI_TRAIN_DATA = None UCI_TEST_DATA = None - +URL_MODEL = 'https://github.com/PaddlePaddle/book/raw/develop/01.fit_a_line/fit_a_line.tar' +MD5_MODEL = '52fc3da8ef3937822fcdd87ee05c0c9b' def feature_range(maximums, minimums): import matplotlib @@ -111,6 +113,13 @@ def test(): return reader +def model(): + tar_file = paddle.v2.dataset.common.download(URL_MODEL, 'fit_a_line.tar', MD5_MODEL) + with open(tar_file, 'r') as f: + parameters = Parameters.from_tar(f) + return parameters + + def fetch(): paddle.v2.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/v2/framework/backward.py b/python/paddle/v2/framework/backward.py index 6827792cb351243f926aeca5f37324dc987d6a79..678efd5d20585355a684bb2df16fdb57a69e0eeb 100644 --- a/python/paddle/v2/framework/backward.py +++ b/python/paddle/v2/framework/backward.py @@ -19,8 +19,20 @@ def append_backward_ops(loss, parameter_list=None, no_grad_set=None): :rtype: list[Variable] """ assert isinstance(loss, framework.Variable) - param_grad_map = loss.block.program.append_backward(loss, no_grad_set or - set()) + + if no_grad_set is None: + program = loss.block.program + assert isinstance(program, framework.Program) + no_grad_set = list() + for block in program.blocks: + assert isinstance(block, framework.Block) + for var in block.vars.itervalues(): + assert isinstance(var, framework.Variable) + if var.stop_gradient: + no_grad_set.append(var.name) + no_grad_set = set(no_grad_set) + + param_grad_map = loss.block.program.append_backward(loss, no_grad_set) if parameter_list is not None: parameters = parameter_list else: diff --git a/python/paddle/v2/framework/evaluator.py b/python/paddle/v2/framework/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..254dd5f1a33eef17ad7a0117541255a4399ef23c --- /dev/null +++ b/python/paddle/v2/framework/evaluator.py @@ -0,0 +1,59 @@ +import paddle.v2.framework.op as op +import numpy as np +import paddle.v2.framework.core as core + + +def avg_accumulate(accumulated_var, per_eval, num_batches, place): + t = np.array(accumulated_var.get_tensor()) + t[0] += per_eval[0] + accumulated_var.get_tensor().set([t[0] / float(num_batches)], place) + + +class Evaluator(object): + def __init__(self, + scope, + operator='accuracy', + input='Inference', + label='Label', + output='Output', + place=core.CPUPlace()): + """ + create an evaluator for evaluating the inference. + NOTE: default run on CPUPlace(), running on GPUPlace doesn't improve performance much. + + :param scope: the scope instance contains the input. + :type scope: paddle.v2.framework.core.scope + :param operator: operator name for caculating the evaluation for each mini-batch. + :type operator: string + :param input: output variable name of forward network. + :type input: string + :param label: variable name of label + :type label: string + """ + self.scope = scope + self.place = place + self.output_name = output + self.num_batches = 0 + # create variable to store accumulated evaluator output + eval_name = ''.join([operator, "@Eval"]) + if scope.find_var(eval_name): + raise Exception("evaluator already exist in scope: %s" % eval_name) + self.accumulated_var = scope.var(eval_name) + t = self.accumulated_var.get_tensor() + t.set_dims((1, )) + t.set([0.0], place) + # self.accumulated_var = block.create_var(block, name=eval_name, shape=(1,)) + # self.accumulated_var.get_tensor().set([0.0]) + # create operator of evaluation + var_map = dict() # var name -> variable + var_map[input] = [input] + var_map[label] = [label] + var_map[output] = [output] + self.op = op.Operator(operator, **var_map) + + def evaluate(self, ctx, accumulator=avg_accumulate): + self.op.run(self.scope, ctx) + per_eval = np.array(self.scope.find_var(self.output_name).get_tensor()) + self.num_batches += 1 + accumulator(self.accumulated_var, per_eval, self.num_batches, + self.place) diff --git a/python/paddle/v2/framework/executor.py b/python/paddle/v2/framework/executor.py index d7d33903ff4f2244eb5365bf7f848c4390c8101b..f5c833190e73a277bef2509e02c4be051768933d 100644 --- a/python/paddle/v2/framework/executor.py +++ b/python/paddle/v2/framework/executor.py @@ -1,5 +1,5 @@ import paddle.v2.framework.core as core -from paddle.v2.framework.framework import Block, Program +from paddle.v2.framework.framework import Block, Program, g_main_program g_scope = core.Scope() @@ -18,7 +18,7 @@ class Executor(object): self.executor = core.Executor(act_places) def run(self, - program, + program=None, feed=None, fetch_list=None, feed_var_name='feed', @@ -29,6 +29,9 @@ class Executor(object): if fetch_list is None: fetch_list = [] + if program is None: + program = g_main_program + if not isinstance(program, Program): raise TypeError() @@ -62,7 +65,7 @@ class Executor(object): outputs={'Out': [fetch_var]}, attrs={'col': i}) - self.executor.run(program.desc, scope, 0) + self.executor.run(program.desc, scope, 0, True) return [ core.get_fetch_variable(scope, fetch_var_name, i) for i in xrange(len(fetch_list)) diff --git a/python/paddle/v2/framework/framework.py b/python/paddle/v2/framework/framework.py index b3493fc378f4d866d16a6a0a739e0ccc8e44d97c..0e6f083e5ba25f4c64fe6988965de93be1d1688b 100644 --- a/python/paddle/v2/framework/framework.py +++ b/python/paddle/v2/framework/framework.py @@ -7,6 +7,19 @@ import copy __all__ = ['Block', 'Variable', 'Program', 'Operator'] +def unique_name(prefix): + uid = core.unique_integer(prefix) # unique during whole process. + return "_".join([prefix, str(uid)]) + + +def _debug_string_(proto): + error_fields = list() + if not proto.IsInitialized(error_fields): + raise ValueError("{0} are not initialized\nThe message is {1}".format( + error_fields, proto)) + return proto.__str__() + + class Variable(object): def __init__(self, block, @@ -16,6 +29,7 @@ class Variable(object): dtype=None, lod_level=None, persistable=None, + stop_gradient=False, **kwargs): self.block = block @@ -84,11 +98,12 @@ class Variable(object): self.block.vars[name] = self self.op = None + self.stop_gradient = stop_gradient def __str__(self): protostr = self.desc.serialize_to_string() proto = framework_pb2.VarDesc.FromString(str(protostr)) - return proto.__str__() + return _debug_string_(proto) __repr__ = __str__ @@ -96,6 +111,10 @@ class Variable(object): def persistable(self): return self.desc.persistable() + @persistable.setter + def persistable(self, p): + self.desc.set_persistable(p) + @property def name(self): return self.desc.name() @@ -264,7 +283,10 @@ class Operator(object): self.desc.set_attr(attr_name, attrs[attr_name]) self.desc.check_attrs() - no_kernel_op_set = {'feed', 'fetch', 'save', 'load'} + no_kernel_op_set = { + 'feed', 'fetch', 'save', 'load', 'recurrent', + 'rnn_memory_helper_grad', 'conditional_block', 'while' + } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) self.desc.infer_shape(self.block.desc) @@ -272,7 +294,7 @@ class Operator(object): def __str__(self): protostr = self.desc.serialize_to_string() proto = framework_pb2.OpDesc.FromString(str(protostr)) - return proto.__str__() + return _debug_string_(proto) __repr__ = __str__ @@ -329,7 +351,7 @@ class Block(object): def __str__(self): protostr = self.desc.serialize_to_string() proto = framework_pb2.BlockDesc.FromString(str(protostr)) - return proto.__str__() + return _debug_string_(proto) __repr__ = __str__ @@ -434,7 +456,7 @@ class Program(object): def __str__(self): protostr = self.desc.serialize_to_string() proto = framework_pb2.ProgramDesc.FromString(str(protostr)) - return proto.__str__() + return _debug_string_(proto) def clone(self): p = Program() @@ -538,5 +560,5 @@ class Parameter(Variable): # program is a global instance. -g_program = Program() -g_init_program = Program() +g_main_program = Program() +g_startup_program = Program() diff --git a/python/paddle/v2/framework/initializer.py b/python/paddle/v2/framework/initializer.py index 507fd16062af1e2458eb9b45407e91a8d29ea9ce..98a87bfa86efb39f381b9f99b2b1f0d7ec7d9833 100644 --- a/python/paddle/v2/framework/initializer.py +++ b/python/paddle/v2/framework/initializer.py @@ -1,6 +1,10 @@ import paddle.v2.framework.framework as framework +import numpy as np -__all__ = ['ConstantInitializer', 'UniformInitializer'] +__all__ = [ + 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', + 'XavierInitializer' +] class Initializer(object): @@ -20,6 +24,41 @@ class Initializer(object): """ raise NotImplementedError() + def _compute_fans(self, var): + """Compute the fan_in and the fan_out for layers + + This method computes the fan_in and the fan_out + for neural network layers, if not specified. It is + not possible to perfectly estimate fan_in and fan_out. + This method will estimate it correctly for matrix multiply and + convolutions. + + Args: + var: variable for which fan_in and fan_out have to be computed + + Returns: + tuple of two integers (fan_in, fan_out) + """ + shape = var.shape + if not shape or len(shape) == 0: + fan_in = fan_out = 1 + elif len(shape) == 1: + fan_in = fan_out = shape[0] + elif len(shape) == 2: + # This is the case for simple matrix multiply + fan_in = shape[0] + fan_out = shape[1] + else: + # Assume this to be a convolutional kernel + # In PaddlePaddle, the shape of the kernel is like: + # [num_filters, num_filter_channels, ...] where the remaining + # dimensions are the filter_size + receptive_field_size = np.prod(shape[2:]) + fan_in = shape[1] * receptive_field_size + fan_out = shape[0] * receptive_field_size + + return (fan_in, fan_out) + class ConstantInitializer(Initializer): """Implements the constant initializer @@ -156,3 +195,93 @@ class NormalInitializer(Initializer): }) var.op = op return op + + +class XavierInitializer(Initializer): + """Implements the Xavier initializer + + This class implements the Xavier weight initializer from the paper + Understanding the difficulty of training deep feedforward neural + networks[1] by Xavier Glorot and Yoshua Bengio. + + This initializer is designed to keep the scale of the gradients + approximately same in all the layers. In case of Uniform distribution, + the range is [-x, x], where x = sqrt(6 / (fan_in + fan_out)). + In case of Normal distribution, the mean is 0 and the standard deviation + is sqrt(2/ (fan_in + fan_out)). + + References: + [1] Understanding the difficulty of training deep feedforward neural + networks. International conference on artificial intelligence and + statistics. + (http://proceedings.mlr.press/v9/glorot10a.html) + """ + + def __init__(self, uniform=True, fan_in=None, fan_out=None, seed=0): + """Constructor for XavierInitializer + + Args: + uniform: whether to use uniform or normal distribution + fan_in: fan_in for Xavier initialization. If None, it is + inferred from the variable. + fan_out: fan_out for Xavier initialization. If None, it is + inferred from the variable. + seed: random seed + + Note: It is recommended to set fan_in and fan_out to None for + most cases. + """ + assert uniform is not None + assert seed is not None + super(XavierInitializer, self).__init__() + self._uniform = uniform + self._fan_in = fan_in + self._fan_out = fan_out + self._seed = seed + + def __call__(self, var, block): + """Add xavier initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + f_in, f_out = self._compute_fans(var) + + # If fan_in and fan_out are passed, use them + fan_in = f_in if self._fan_in is None else self._fan_in + fan_out = f_out if self._fan_out is None else self._fan_out + + if self._uniform: + limit = np.sqrt(6.0 / float(fan_in + fan_out)) + op = block.prepend_op( + type="uniform_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "min": -limit, + "max": limit, + "seed": self._seed + }) + + else: + std = np.sqrt(2.0 / float(fan_in + fan_out)) + op = block.prepend_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "mean": 0.0, + "std": std, + "seed": self._seed + }) + var.op = op + return op diff --git a/python/paddle/v2/framework/io.py b/python/paddle/v2/framework/io.py index f3ba719bde086f696a27b806228a8c97466a681e..5c247904a330e25b1a9f53db431947840db3f615 100644 --- a/python/paddle/v2/framework/io.py +++ b/python/paddle/v2/framework/io.py @@ -1,7 +1,7 @@ import os import cPickle as pickle -from paddle.v2.framework.framework import Program, Parameter, g_program, \ +from paddle.v2.framework.framework import Program, Parameter, g_main_program, \ Variable __all__ = [ @@ -29,13 +29,13 @@ def _clone_var_in_block_(block, var): persistable=True) -def save_vars(executor, dirname, program=None, vars=None, predicate=None): +def save_vars(executor, dirname, main_program=None, vars=None, predicate=None): """ Save variables to directory by executor. :param executor: executor that save variable :param dirname: directory path - :param program: program. If vars is None, then filter all variables in this + :param main_program: program. If vars is None, then filter all variables in this program which fit `predicate`. Default g_program. :param predicate: The Predicate describes a callable that returns a variable as a bool. If it returns true, the variables will be saved. @@ -44,15 +44,15 @@ def save_vars(executor, dirname, program=None, vars=None, predicate=None): :return: None """ if vars is None: - if program is None: - program = g_program - if not isinstance(program, Program): + if main_program is None: + main_program = g_main_program + if not isinstance(main_program, Program): raise TypeError("program should be as Program type or None") save_vars( executor, dirname=dirname, - vars=filter(predicate, program.list_vars())) + vars=filter(predicate, main_program.list_vars())) else: save_program = Program() save_block = save_program.global_block() @@ -66,37 +66,37 @@ def save_vars(executor, dirname, program=None, vars=None, predicate=None): executor.run(save_program) -def save_params(executor, dirname, program=None): +def save_params(executor, dirname, main_program=None): """ Save all parameters to directory with executor. """ save_vars( executor, dirname=dirname, - program=program, + main_program=main_program, vars=None, predicate=is_parameter) -def save_persistables(executor, dirname, program=None): +def save_persistables(executor, dirname, main_program=None): """ Save all persistables to directory with executor. """ save_vars( executor, dirname=dirname, - program=program, + main_program=main_program, vars=None, predicate=is_persistable) -def load_vars(executor, dirname, program=None, vars=None, predicate=None): +def load_vars(executor, dirname, main_program=None, vars=None, predicate=None): """ Load variables from directory by executor. :param executor: executor that save variable :param dirname: directory path - :param program: program. If vars is None, then filter all variables in this + :param main_program: program. If vars is None, then filter all variables in this program which fit `predicate`. Default g_program. :param predicate: The Predicate describes a callable that returns a variable as a bool. If it returns true, the variables will be loaded. @@ -105,15 +105,15 @@ def load_vars(executor, dirname, program=None, vars=None, predicate=None): :return: None """ if vars is None: - if program is None: - program = g_program - if not isinstance(program, Program): + if main_program is None: + main_program = g_main_program + if not isinstance(main_program, Program): raise TypeError("program's type should be Program") load_vars( executor, dirname=dirname, - vars=filter(predicate, program.list_vars())) + vars=filter(predicate, main_program.list_vars())) else: load_prog = Program() load_block = load_prog.global_block() @@ -129,27 +129,33 @@ def load_vars(executor, dirname, program=None, vars=None, predicate=None): executor.run(load_prog) -def load_params(executor, dirname, program=None): +def load_params(executor, dirname, main_program=None): """ load all parameters from directory by executor. """ load_vars( - executor, dirname=dirname, program=program, predicate=is_parameter) + executor, + dirname=dirname, + main_program=main_program, + predicate=is_parameter) -def load_persistables(executor, dirname, program=None): +def load_persistables(executor, dirname, main_program=None): """ load all persistables from directory by executor. """ load_vars( - executor, dirname=dirname, program=program, predicate=is_persistable) + executor, + dirname=dirname, + main_program=main_program, + predicate=is_persistable) def save_inference_model(dirname, feeded_var_names, target_vars, executor, - program=None): + main_program=None): """ Build a model especially for inference, and save it to directory by the executor. @@ -158,20 +164,20 @@ def save_inference_model(dirname, :param feeded_var_names: Names of variables that need to be feeded data during inference :param target_vars: Variables from which we can get inference results. :param executor: executor that save inference model - :param program: original program, which will be pruned to build the inference model. + :param main_program: original program, which will be pruned to build the inference model. Default g_program. :return: None """ - if program is None: - program = g_program + if main_program is None: + main_program = g_main_program if not isinstance(target_vars, list): target_vars = [target_vars] if not os.path.isdir(dirname): os.makedirs(dirname) - pruned_program = program.prune(target_vars) + pruned_program = main_program.prune(target_vars) fetch_var_names = [v.name for v in target_vars] model_file_name = dirname + "/__model__" @@ -182,10 +188,10 @@ def save_inference_model(dirname, "fetch_var_names": fetch_var_names }, f, -1) - save_params(executor, dirname, program) + save_params(executor, dirname, main_program) -def load_persistables_if_exist(executor, dirname, program=None): +def load_persistables_if_exist(executor, dirname, main_program=None): filenames = next(os.walk(dirname))[2] filenames = set(filenames) @@ -198,7 +204,7 @@ def load_persistables_if_exist(executor, dirname, program=None): load_vars( executor, dirname, - program=program, + main_program=main_program, vars=None, predicate=_is_presistable_and_exist_) diff --git a/python/paddle/v2/framework/layer_helper.py b/python/paddle/v2/framework/layer_helper.py index 45d9cf3f4827a3c31005f9650240bcdba47696e7..552976185dfc2ece8689ae4dceb3bb3a68a27ea7 100644 --- a/python/paddle/v2/framework/layer_helper.py +++ b/python/paddle/v2/framework/layer_helper.py @@ -1,17 +1,10 @@ import copy import itertools -import paddle.v2.framework.core as core - -from paddle.v2.framework.framework import Variable, g_program, \ - g_init_program +from paddle.v2.framework.framework import Variable, g_main_program, \ + g_startup_program, unique_name, Program from paddle.v2.framework.initializer import ConstantInitializer, \ - UniformInitializer - - -def unique_name(prefix): - uid = core.unique_integer(prefix) # unique during whole process. - return "_".join([prefix, str(uid)]) + UniformInitializer, XavierInitializer class LayerHelper(object): @@ -27,23 +20,23 @@ class LayerHelper(object): return self.kwargs['name'] @property - def program(self): - prog = self.kwargs.get('program', None) + def main_program(self): + prog = self.kwargs.get('main_program', None) if prog is None: - return g_program + return g_main_program else: return prog @property - def init_program(self): - prog = self.kwargs.get('init_program', None) + def startup_program(self): + prog = self.kwargs.get('startup_program', None) if prog is None: - return g_init_program + return g_startup_program else: return prog def append_op(self, *args, **kwargs): - return self.program.current_block().append_op(*args, **kwargs) + return self.main_program.current_block().append_op(*args, **kwargs) def multiple_input(self, input_param_name='input'): inputs = self.kwargs.get(input_param_name, []) @@ -68,7 +61,7 @@ class LayerHelper(object): @property def param_attr(self): - default = {'name': None, 'initializer': UniformInitializer()} + default = {'name': None, 'initializer': XavierInitializer()} actual = self.kwargs.get('param_attr', None) if actual is None: actual = default @@ -77,10 +70,11 @@ class LayerHelper(object): actual[default_field] = default[default_field] return actual + @property def bias_attr(self): - default = {'name': None, 'initializer': ConstantInitializer()} + default = {'name': None, 'initializer': XavierInitializer()} bias_attr = self.kwargs.get('bias_attr', None) - if bias_attr is True: + if bias_attr is None: bias_attr = default if isinstance(bias_attr, dict): @@ -119,28 +113,41 @@ class LayerHelper(object): raise ValueError("Data Type mismatch") return dtype - def create_parameter(self, attr, shape, dtype, suffix='w'): + def create_parameter(self, attr, shape, dtype, suffix='w', + initializer=None): # Deepcopy the attr so that parameters can be shared in program attr_copy = copy.deepcopy(attr) + if initializer is not None: + attr_copy['initializer'] = initializer if attr_copy['name'] is None: attr_copy['name'] = unique_name(".".join([self.name, suffix])) - self.init_program.global_block().create_parameter( + self.startup_program.global_block().create_parameter( dtype=dtype, shape=shape, **attr_copy) - return self.program.global_block().create_parameter( + return self.main_program.global_block().create_parameter( name=attr_copy['name'], dtype=dtype, shape=shape) def create_tmp_variable(self, dtype): - return self.program.current_block().create_var( + return self.main_program.current_block().create_var( name=unique_name(".".join([self.name, 'tmp'])), dtype=dtype, persistable=False) def create_variable(self, *args, **kwargs): - return self.program.current_block().create_var(*args, **kwargs) - - def create_global_variable(self, *args, **kwargs): - return self.program.global_block().create_var( - *args, persistable=False, **kwargs) + return self.main_program.current_block().create_var(*args, **kwargs) + + def create_global_variable(self, persistable=False, *args, **kwargs): + return self.main_program.global_block().create_var( + *args, persistable=persistable, **kwargs) + + def set_variable_initializer(self, var, initializer): + assert isinstance(var, Variable) + self.startup_program.global_block().create_var( + name=var.name, + type=var.type, + dtype=var.data_type, + shape=var.shape, + persistable=True, + initializer=initializer) def append_bias_op(self, input_var, num_flatten_dims=None): """ @@ -160,7 +167,7 @@ class LayerHelper(object): num_flatten_dims = 1 size = list(input_var.shape[num_flatten_dims:]) - bias_attr = self.bias_attr() + bias_attr = self.bias_attr if not bias_attr: return input_var diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py index 86a2c7bf08b09638fd065f96300fc4f7ffc332d5..ae85f460f78c9a661d490e0fe673882bcbacd19f 100644 --- a/python/paddle/v2/framework/layers.py +++ b/python/paddle/v2/framework/layers.py @@ -1,38 +1,65 @@ -from paddle.v2.framework.layer_helper import LayerHelper, unique_name import paddle.v2.framework.core as core -from paddle.v2.framework.framework import OpProtoHolder, Variable, Program -from paddle.v2.framework.initializer import ConstantInitializer +import paddle.v2.framework.proto.framework_pb2 as framework_pb2 +from paddle.v2.framework.framework import OpProtoHolder, Variable, Program, \ + Operator +from paddle.v2.framework.initializer import ConstantInitializer, \ + NormalInitializer +from paddle.v2.framework.layer_helper import LayerHelper, unique_name import re +import cStringIO __all__ = [ 'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat', 'StaticRNN', 'cast', 'sequence_conv', 'sequence_pool', 'sums', 'cos_sim', - 'batch_norm', 'accuracy' + 'batch_norm', 'accuracy', 'split_lod_tensor' ] def fc(input, size, param_attr=None, - bias_attr=True, + bias_attr=None, name=None, act=None, num_flatten_dims=1, - program=None, - init_program=None): - # create helper + main_program=None, + startup_program=None): + """ + Fully Connected Layer. + + Args: + input: The input tensor to the function + size: The size of the layer + param_attr: The parameters/weights to the FC Layer + bias_attr: The bias parameter for the FC layer + name: Name/alias of the function + act: Activation to be applied to the output of FC layer + num_flatten_dims: Number of columns in input + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in multiple inputs and performs the Fully Connected + function (linear transformation) on top of each of them. + So for input x, the output will be : Wx + b. Where W is the parameter, + b the bias and x is the input. + + The function also applies an activation (non-linearity) on top of the + output, if activation is passed in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('fc', **locals()) dtype = helper.input_dtype() - # mul mul_results = [] for input_var, param_attr in helper.iter_inputs_and_params(): input_shape = input_var.shape param_shape = [ reduce(lambda a, b: a * b, input_shape[num_flatten_dims:], 1) ] + [size] - w = helper.create_parameter( attr=param_attr, shape=param_shape, dtype=dtype) tmp = helper.create_tmp_variable(dtype) @@ -65,8 +92,28 @@ def embedding(input, data_type='float32', is_sparse=False, param_attr=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): + """ + Embedding Layer. + + Args: + input: The input to the function + size: The size of the layer + data_type: The type of data : float32, float_16, int etc + is_sparse: A flag that decleares whether the input is sparse + param_attr: Parameters for this layer + main_program: Name of the main program that calls this + startup_program: Name of the startup program + + This function can take in the input (which is a vector of IDs) and + performs a lookup in the lookup_table using these IDs, to result into + the embedding of each ID in the input. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('embedding', **locals()) w = helper.create_parameter( attr=helper.param_attr, shape=size, dtype=data_type) @@ -80,26 +127,189 @@ def embedding(input, return tmp +# TODO(qijun): expose H0 and C0 +def dynamic_lstm(input, + size, + data_type='float32', + param_attr=None, + bias_attr=None, + use_peepholes=True, + is_reverse=False, + gate_activation='sigmoid', + cell_activation='tanh', + candidate_activation='tanh', + main_program=None, + startup_program=None): + helper = LayerHelper('lstm', **locals()) + size = size / 4 + weight = helper.create_parameter( + attr=helper.param_attr, shape=[size, 4 * size], dtype=data_type) + bias_size = [1, 7 * size] + if not use_peepholes: + bias_size[1] = 4 * size + bias = helper.create_parameter( + attr=helper.bias_attr, shape=bias_size, dtype=data_type, suffix='b') + + hidden = helper.create_tmp_variable(data_type) + cell = helper.create_tmp_variable(data_type) + batch_gate = helper.create_tmp_variable(data_type) + batch_cell_pre_act = helper.create_tmp_variable(data_type) + + helper.append_op( + type='lstm', + inputs={'Input': input, + 'Weight': weight, + 'Bias': bias}, + outputs={ + 'Hidden': hidden, + 'Cell': cell, + 'BatchGate': batch_gate, + 'BatchCellPreAct': batch_cell_pre_act + }, + attrs={ + 'use_peepholes': use_peepholes, + 'is_reverse': is_reverse, + 'gate_activation': gate_activation, + 'cell_activation': cell_activation, + 'candidate_activation': candidate_activation + }) + return hidden, cell + + def data(name, shape, data_type='float32', type=core.VarDesc.VarType.LOD_TENSOR, append_batch_size=True, - program=None, - init_program=None): + main_program=None, + startup_program=None, + stop_gradient=True): + """ + Data Layer. + + Args: + name: The name/alias of the function + shape: Tuple declaring the shape. + data_type: The type of data : float32, float_16, int etc + type: The output type. By default it is LOD_TENSOR. + append_batch_size: Whether or not to append the data as a batch. + main_program: Name of the main program that calls this + startup_program: Name of the startup program + stop_gradient: A boolean that mentions whether gradient should flow. + + This function takes in input and based on whether data has + to be returned back as a minibatch, it creates the global variable using + the helper functions. The global variables can be accessed by all the + following operations and layers in the graph. + + All the input variables of this function are passed in as local variables + to the LayerHelper constructor. + + """ helper = LayerHelper('data', **locals()) + shape = list(shape) + for i in xrange(len(shape)): + if shape[i] is None: + shape[i] = -1 + append_batch_size = False + elif shape[i] < 0: + append_batch_size = False + if append_batch_size: shape = [-1] + shape # append batch size as -1 + return helper.create_global_variable( - name=name, shape=shape, dtype=data_type, type=type) + name=name, + shape=shape, + dtype=data_type, + type=type, + stop_gradient=stop_gradient) + + +def create_tensor(dtype, name=None, main_program=None): + helper = LayerHelper("create_tensor", **locals()) + return helper.create_variable(name=helper.name, dtype=dtype) def _convert_(name): + """ + Formatting. + + Args: + name: The name/alias + + This function takes in a name and converts it to a standard format of + group1_group2. Where as per the regular expression, group1 can have + alphabets and numbers and group2 has capital alphabets. + + """ s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() +def _generate_doc_string_(op_proto): + """ + Generate docstring by OpProto + + Args: + op_proto (framework_pb2.OpProto): a protobuf message typed OpProto + + Returns: + str: the document string + """ + + def _type_to_str_(tp): + return framework_pb2.AttrType.Name(tp) + + if not isinstance(op_proto, framework_pb2.OpProto): + raise TypeError("OpProto should be `framework_pb2.OpProto`") + + buf = cStringIO.StringIO() + buf.write(op_proto.comment) + buf.write('\nArgs:\n') + for each_input in op_proto.inputs: + line_begin = ' {0}: '.format(_convert_(each_input.name)) + buf.write(line_begin) + buf.write(each_input.comment) + buf.write('\n') + buf.write(' ' * len(line_begin)) + buf.write('Duplicable: ') + buf.write(str(each_input.duplicable)) + buf.write(' Optional: ') + buf.write(str(each_input.dispensable)) + buf.write('\n') + + for each_attr in op_proto.attrs: + buf.write(' ') + buf.write(each_attr.name) + buf.write(' (') + buf.write(_type_to_str_(each_attr.type)) + buf.write('): ') + buf.write(each_attr.comment) + buf.write('\n') + + if len(op_proto.outputs) != 0: + buf.write('\nReturns:\n') + buf.write(' ') + for each_opt in op_proto.outputs: + if not each_opt.intermediate: + break + buf.write(each_opt.comment) + + return buf.getvalue() + + def _create_op_func_(op_type): + """ + Create an Operator for a Function. + + Args: + op_type: The name of the operator to be created + + This function takes in the operator type (sigmoid, mean , average etc) and + creates the operator functionality. + + """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) not_intermediate_outputs = \ filter(lambda output: not output.intermediate, op_proto.outputs) @@ -107,26 +317,26 @@ def _create_op_func_(op_type): filter(lambda output: output.intermediate, op_proto.outputs) if len(not_intermediate_outputs) != 1: - raise ValueError( - "Only one not intermediate output operator can be automatically generated" - ) + raise ValueError("Only one non intermediate output operator can be", + "automatically generated") if not_intermediate_outputs[0].duplicable: raise ValueError( - "Only not duplicable op can be automatically generated") + "Only non duplicable op can be automatically generated") for output in intermediate_outputs: if output.duplicable: - raise ValueError( - "Only when all intermediate ops are not duplicable, " - "this op can be automatically generated") + raise ValueError("The op can be automatically generated only when ", + "all intermediate ops are not duplicable") o_name = not_intermediate_outputs[0].name intermediate_output_names = [output.name for output in intermediate_outputs] - def func(**kwargs): - helper = LayerHelper(op_type, **kwargs) - inputs = dict() + def infer_and_check_data_type(op_proto, **kwargs): + """ + This function performs the sanity check for data_type and + instance type. + """ dtype = None for ipt in op_proto.inputs: name = _convert_(ipt.name) @@ -143,6 +353,20 @@ def _create_op_func_(op_type): elif dtype != each.data_type: raise ValueError( "operator {0} must input same dtype".format(op_type)) + + return dtype + + def func(**kwargs): + helper = LayerHelper(op_type, **kwargs) + + dtype = infer_and_check_data_type(op_proto, **kwargs) + + inputs = dict() + for ipt in op_proto.inputs: + name = _convert_(ipt.name) + val = kwargs.pop(name, []) + if not isinstance(val, list) and not isinstance(val, tuple): + val = [val] inputs[ipt.name] = val outputs = dict() @@ -156,6 +380,7 @@ def _create_op_func_(op_type): func.__name__ = op_type globals()[op_type] = func + func.__doc__ = _generate_doc_string_(op_proto) global __all__ __all__.append(op_type) @@ -165,9 +390,35 @@ _create_op_func_('mul') _create_op_func_('elementwise_add') _create_op_func_('dropout') _create_op_func_('reshape') +_create_op_func_('elementwise_add') +_create_op_func_('sigmoid') +_create_op_func_('scale') +_create_op_func_('reshape') +_create_op_func_('transpose') + + +def fill_constant(data_type, shape, value=None, program=None): + """ + This function creates a tensor , with shape as mentioned in the input and + specified data_type and fills this up with a constant value that + comes in the input. + """ + helper = LayerHelper('fill_constant', **locals()) + out = helper.create_tmp_variable(dtype=data_type) + helper.append_op( + type='fill_constant', + outputs={'Out': [out]}, + attrs={'data_type': data_type, + 'shape': shape, + 'value': value}) + return out -def cast(x, data_type, program=None): +def cast(x, data_type, main_program=None): + """ + This function takes in the input with input_data_type + and casts it to the output_data_type as the output. + """ helper = LayerHelper('cast', **locals()) out = helper.create_tmp_variable(dtype=data_type) helper.append_op( @@ -179,7 +430,11 @@ def cast(x, data_type, program=None): return out -def concat(input, axis, program=None, init_program=None): +def concat(input, axis, main_program=None, startup_program=None): + """ + This function concats the input along the axis mentioned + and returns that as the output. + """ helper = LayerHelper('concat', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) helper.append_op( @@ -190,18 +445,76 @@ def concat(input, axis, program=None, init_program=None): return out -def sums(input, program=None, init_program=None): +def sums(input, main_program=None, startup_program=None): + """ + This function takes in the input and performs the sum operation on it + and returns that as the output. + """ helper = LayerHelper('sum', **locals()) out = helper.create_tmp_variable(dtype=helper.input_dtype()) - helper.append_op(type='sum', inputs={'X': [input]}, outputs={'Out': out}) + helper.append_op(type='sum', inputs={'X': input}, outputs={'Out': out}) return out -def cos_sim(X, Y, program=None, init_program=None): - helper = LayerHelper('cos_sim', **locals()) - out = helper.create_tmp_variable(dtype=helper.input_dtype("X")) - xnorm = helper.create_tmp_variable(dtype=helper.input_dtype("X")) - ynorm = helper.create_tmp_variable(dtype=helper.input_dtype("X")) +def assign(input, output, main_program=None): + helper = LayerHelper('assign', **locals()) + helper.append_op( + type='scale', + inputs={'X': [input]}, + outputs={'Out': [output]}, + attrs={'scale': 1.0}) + return output + + +def split_lod_tensor(input, + mask, + level, + main_program=None, + startup_program=None): + helper = LayerHelper('split_lod_tensor', **locals()) + out_true = helper.create_tmp_variable(dtype=input.data_type) + out_false = helper.create_tmp_variable(dtype=input.data_type) + helper.append_op( + type='split_lod_tensor', + inputs={ + 'X': input, + 'Mask': mask, + }, + outputs={'OutTrue': out_true, + 'OutFalse': out_false}, + attrs={'level': level}) + return out_true, out_false + + +def merge_lod_tensor(in_true, + in_false, + x, + mask, + level, + main_program=None, + startup_program=None): + helper = LayerHelper('merge_lod_tensor', **locals()) + out = helper.create_tmp_variable(dtype=x.data_type) + helper.append_op( + type='merge_lod_tensor', + inputs={'X': x, + 'Mask': mask, + 'InTrue': in_true, + 'InFalse': in_false}, + outputs={'Out': out}, + attrs={'level': level}) + return out + + +def cos_sim(X, Y, **kwargs): + """ + This function performs the cosine similarity between two tensors + X and Y and returns that as the output. + """ + helper = LayerHelper('cos_sim', **kwargs) + out = helper.create_tmp_variable(dtype=X.data_type) + xnorm = helper.create_tmp_variable(dtype=X.data_type) + ynorm = helper.create_tmp_variable(dtype=X.data_type) helper.append_op( type='cos_sim', inputs={'X': [X], @@ -209,10 +522,13 @@ def cos_sim(X, Y, program=None, init_program=None): outputs={'Out': [out], 'XNorm': [xnorm], 'YNorm': [ynorm]}) - return out, xnorm, ynorm + return out def cross_entropy(input, label, **kwargs): + """ + This function computes cross_entropy using the input and label. + """ helper = LayerHelper('cross_entropy', **kwargs) out = helper.create_tmp_variable(dtype=input.data_type) helper.append_op( @@ -225,6 +541,10 @@ def cross_entropy(input, label, **kwargs): def square_error_cost(input, label, **kwargs): + """ + This functions returns the squared error cost using the input and label. + The output is appending the op to do the above. + """ helper = LayerHelper('square_error_cost', **kwargs) minus_out = helper.create_tmp_variable(dtype=input.data_type) helper.append_op( @@ -240,6 +560,10 @@ def square_error_cost(input, label, **kwargs): def accuracy(input, label, k=1, **kwargs): + """ + This function computes the accuracy using the input and label. + The output is the top_k inputs and their indices. + """ helper = LayerHelper("accuracy", **kwargs) topk_out = helper.create_tmp_variable(dtype=input.data_type) topk_indices = helper.create_tmp_variable(dtype="int64") @@ -265,12 +589,18 @@ def accuracy(input, label, k=1, **kwargs): def sequence_conv(input, num_filters, filter_size=3, - stride=1, + filter_stride=1, + act=None, padding=None, bias_attr=None, param_attr=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): + """ + This function creates the op for sequence_conv, using the inputs and + other convolutional configurations for the filters and stride as given + in the input parameters to the function. + """ # FIXME(dzh) : want to unify the argument of python layer # function. So we ignore some unecessary attributes. # such as, padding_trainable, context_start. @@ -291,9 +621,9 @@ def sequence_conv(input, }, outputs={"Out": pre_bias}, attrs={ - 'context_stride': stride, - 'context_start': 0, - 'context_length': filter_size + 'contextStride': filter_stride, + 'contextStart': -int(filter_size / 2), + 'contextLength': filter_size }) pre_act = helper.append_bias_op(pre_bias) return helper.append_activation(pre_act) @@ -309,8 +639,15 @@ def conv2d(input, padding=None, bias_attr=None, param_attr=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): + """ + This function creates the op for a 2-dimensional Convolution. + This is performed using the parameters of filters(size, dimensionality etc) + , stride and other configurations for a Convolution operation. + This funciton can also append an activation on top of the + conv-2d output, if mentioned in the input parameters. + """ helper = LayerHelper('conv2d', **locals()) dtype = helper.input_dtype() @@ -331,8 +668,13 @@ def conv2d(input, input_shape = input.shape filter_shape = [num_filters, num_filter_channels] + filter_size + + std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 filter = helper.create_parameter( - attr=helper.param_attr, shape=filter_shape, dtype=dtype) + attr=helper.param_attr, + shape=filter_shape, + dtype=dtype, + initializer=NormalInitializer(0.0, std, 0)) pre_bias = helper.create_tmp_variable(dtype) helper.append_op( @@ -346,25 +688,27 @@ def conv2d(input, 'paddings': padding, 'groups': groups}) - pre_act = helper.append_bias_op(pre_bias) + pre_act = helper.append_bias_op(pre_bias, 1) return helper.append_activation(pre_act) def sequence_pool(input, pool_type, **kwargs): - ENUM_POOL_TYPE = set(["MAX", "AVG", "SQRT", "LAST", "FIRST"]) - if pool_type.upper() not in ENUM_POOL_TYPE: - raise ValueError("Unknown pool_type: '%s'. It can only be %s.", - str(pool_type), " ".join(ENUM_POOL_TYPE)) - - helper = LayerHelper('sequence_pool', **kwargs) + """ + This function add the operator for sequence pooling. + This is applied on top of the input using pool_type mentioned + in the parameters. + """ + helper = LayerHelper('sequence_pool', input=input, **kwargs) dtype = helper.input_dtype() pool_out = helper.create_tmp_variable(dtype) + max_index = helper.create_tmp_variable(dtype) helper.append_op( type="sequence_pool", - inputs={"X": [input]}, - outputs={"Out": [pool_out]}, + inputs={"X": input}, + outputs={"Out": pool_out, + "MaxIndex": max_index}, attrs={"pooltype": pool_type.upper()}) return pool_out @@ -376,8 +720,12 @@ def pool2d(input, pool_stride=[1, 1], pool_padding=[0, 0], global_pooling=False, - program=None, - init_program=None): + main_program=None, + startup_program=None): + """ + This function adds the operator for pooling in 2 dimensions, using the + pooling configurations mentioned in input parameters. + """ if pool_type not in ["max", "avg"]: raise ValueError( "Unknown pool_type: '%s'. It can only be 'max' or 'avg'.", @@ -398,9 +746,9 @@ def pool2d(input, inputs={"X": input}, outputs={"Out": pool_out}, attrs={ - "poolingType": pool_type, + "pooling_type": pool_type, "ksize": pool_size, - "globalPooling": global_pooling, + "global_pooling": global_pooling, "strides": pool_stride, "paddings": pool_padding }) @@ -412,12 +760,16 @@ def batch_norm(input, act=None, is_test=False, momentum=0.9, - epsilon=1e05, + epsilon=1e-05, param_attr=None, bias_attr=None, data_layout='NCHW', - program=None, - init_program=None): + main_program=None, + startup_program=None): + """ + This function helps create an operator to implement + the BatchNorm layer using the configurations from the input parameters. + """ helper = LayerHelper('batch_norm', **locals()) dtype = helper.input_dtype() @@ -430,27 +782,29 @@ def batch_norm(input, else: raise ValueError("unsupported data layout:" + data_layout) - def create_persistable_var(dtype, shape, initializer=None): - name = unique_name(".".join([helper.name, "xxxx"])) - var = init_program.global_block().create_var( - dtype=dtype, shape=shape, name=name, persistable=True) - if initializer is not None: - initializer(var, var.block) - return program.global_block().create_var( - name=name, dtype=dtype, shape=shape, persistable=True) - param_shape = [channel_num] # create parameter scale = helper.create_parameter( - attr=helper.param_attr, shape=param_shape, dtype=dtype) + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + initializer=ConstantInitializer(1.0)) bias = helper.create_parameter( - attr=helper.param_attr, shape=param_shape, dtype=dtype) + attr=helper.param_attr, + shape=param_shape, + dtype=dtype, + initializer=ConstantInitializer(0.0)) + + mean = helper.create_global_variable( + dtype=input.data_type, shape=param_shape, persistable=True) + helper.set_variable_initializer( + var=mean, initializer=ConstantInitializer(0.0)) - # create input - mean = create_persistable_var(dtype, param_shape, ConstantInitializer(0.0)) - variance = create_persistable_var(dtype, param_shape, - ConstantInitializer(1.0)) + variance = helper.create_global_variable( + dtype=input.data_type, shape=param_shape, persistable=True) + helper.set_variable_initializer( + var=variance, initializer=ConstantInitializer(1.0)) # create output # mean and mean_out share the same memory @@ -487,30 +841,38 @@ def batch_norm(input, class BlockGuard(object): """ - BlockGuard used to create sub-block in program by using Python `with` - keyword. + BlockGuard class. + + BlockGuard class is used to create a sub-block in a program by + using the Python `with` keyword. """ - def __init__(self, program): - if not isinstance(program, Program): + def __init__(self, main_program): + if not isinstance(main_program, Program): raise TypeError("BlockGuard takes a program") - self.program = program + self.main_program = main_program def __enter__(self): - self.program.create_block() + self.main_program.create_block() def __exit__(self, exc_type, exc_val, exc_tb): - self.program.rollback() + self.main_program.rollback() if exc_type is not None: return False # re-raise exception return True class StaticRNNGuard(BlockGuard): + """ + StaticRNNGuard class. + + StaticRNNGuard class is used to create a StaticRNN block in a program. + """ + def __init__(self, rnn): if not isinstance(rnn, StaticRNN): - raise TypeError("StaticRNNGuard takes an StaticRNN") - super(StaticRNNGuard, self).__init__(rnn.helper.program) + raise TypeError("StaticRNNGuard takes a StaticRNN") + super(StaticRNNGuard, self).__init__(rnn.helper.main_program) self.rnn = rnn def __enter__(self): @@ -518,6 +880,8 @@ class StaticRNNGuard(BlockGuard): return super(StaticRNNGuard, self).__enter__() def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + return False self.rnn.status = StaticRNN.AFTER_RNN_BLOCK self.rnn.complete_rnn_op() return super(StaticRNNGuard, self).__exit__(exc_type, exc_val, exc_tb) @@ -525,12 +889,18 @@ class StaticRNNGuard(BlockGuard): class StaticRNNMemoryLink(object): """ - :param init: the initial variable for Memory - :type init: Variable - :param pre_mem: the memory variable in previous time step - :type pre_mem: Variable - :param mem: the memory variable in current time step - :type mem: Variable + StaticRNNMemoryLink class. + + Args: + init: the initial variable for Memory + init: Variable + pre_mem: the memory variable in previous time step + pre_mem: Variable + mem: the memory variable in current time step + mem: Variable + + StaticRNNMemoryLink class is used to create a link between two + memory cells of a StaticRNN. """ def __init__(self, init, pre_mem, mem=None): @@ -540,12 +910,19 @@ class StaticRNNMemoryLink(object): class StaticRNN(object): + """ + StaticRNN class. + + StaticRNN class is used to create a StaticRNN. The RNN will have its + own parameters like inputs, outputs, memories, status and length. + """ BEFORE_RNN_BLOCK = 0 IN_RNN_BLOCK = 1 AFTER_RNN_BLOCK = 2 - def __init__(self, name=None, program=None): - self.helper = LayerHelper("static_rnn", name=name, program=program) + def __init__(self, name=None, main_program=None): + self.helper = LayerHelper( + "static_rnn", name=name, main_program=main_program) self.memories = {} # memory map, from pre_mem.name --> MemoryLink self.inputs = [] # input variable list in current block self.outputs = [] # output variable list in parent block @@ -560,25 +937,45 @@ class StaticRNN(object): if self.status != StaticRNN.IN_RNN_BLOCK: raise ValueError("You must invoke {0} in rnn block".format(method)) - def memory(self, init=None, shape=None, dtype=None, init_value=0): + def memory(self, + init=None, + shape=None, + batch_ref=None, + init_value=0.0, + init_batch_dim_idx=0, + ref_batch_dim_idx=1): + """ + Args: + init: boot memory, if not set, a shape, batch_ref must be provided + shape: shape of the boot memory + batch_ref: batch size reference variable + init_value: the init value of boot memory + init_batch_dim_idx: the index of batch size in init's dimension + ref_batch_dim_idx: the index of batch size in batch_ref's dimension + """ self._assert_in_rnn_block_('memory') if init is None: - if shape is None or dtype is None: + if shape is None or batch_ref is None: raise ValueError( - "if init is None, memory at least need shape and dtype") + "if init is None, memory at least need shape and batch_ref") parent_block = self.parent_block() var_name = unique_name("@".join([self.helper.name, "memory_boot"])) boot_var = parent_block.create_var( - name=var_name, shape=shape, dtype=dtype, persistable=False) + name=var_name, + shape=shape, + dtype=batch_ref.data_type, + persistable=False) parent_block.append_op( - type="fill_constant", - inputs={}, + type="fill_constant_batch_size_like", + inputs={'Input': [batch_ref]}, outputs={'Out': [boot_var]}, attrs={ 'value': init_value, 'shape': boot_var.shape, - 'data_type': boot_var.data_type + 'data_type': boot_var.data_type, + 'input_dim_idx': ref_batch_dim_idx, + 'output_dim_idx': init_batch_dim_idx }) return self.memory(init=boot_var) @@ -596,14 +993,14 @@ class StaticRNN(object): if not isinstance(x, Variable): raise TypeError("step input takes a Variable") if self.seq_len is None: - self.seq_len = x.shape[1] - elif self.seq_len != x.shape[1]: + self.seq_len = x.shape[0] + elif self.seq_len != x.shape[0]: raise ValueError("Static RNN only take fix seq_len input") ipt = self.helper.create_variable( name=x.name, dtype=x.data_type, - shape=[-1] + list(x.shape[2:]), + shape=list(x.shape[1:]), type=x.type) self.inputs.append(ipt) return ipt @@ -613,10 +1010,17 @@ class StaticRNN(object): if not isinstance(o, Variable): raise TypeError("step output takes a Variable") + tmp_o = self.helper.create_tmp_variable(dtype=o.data_type) + self.helper.append_op( + type='rnn_memory_helper', + inputs={'X': [o]}, + outputs={'Out': tmp_o}, + attrs={'data_type': o.data_type}) + out_var = self.parent_block().create_var( - name=o.name, - shape=[-1, self.seq_len] + list(o.shape[1:]), - dtype=o.data_type) + name=tmp_o.name, + shape=[self.seq_len] + list(tmp_o.shape), + dtype=tmp_o.data_type) self.outputs.append(out_var) @@ -630,7 +1034,7 @@ class StaticRNN(object): self.memories[mem.name].mem = var def parent_block(self): - prog = self.helper.program + prog = self.helper.main_program parent_idx = prog.current_block().parent_idx assert parent_idx >= 0 parent_block = prog.block(parent_idx) @@ -647,6 +1051,452 @@ class StaticRNN(object): return self.outputs def complete_rnn_op(self): - # TODO(yuyang18): Create RNN Op here. - # Implement this method after RNN op complete. - pass + main_program = self.helper.main_program + rnn_block = main_program.current_block() + parent_block = self.parent_block() + + local_inputs = set() + + for op in rnn_block.ops: + assert isinstance(op, Operator) + for oname in op.output_names: + for out_var_name in op.output(oname): + local_inputs.add(out_var_name) + + for var in self.inputs: + local_inputs.add(var.name) + for m in self.memories: + local_inputs.add(m) + + params = list() + for op in rnn_block.ops: + assert isinstance(op, Operator) + for iname in op.input_names: + for in_var_name in op.input(iname): + if in_var_name not in local_inputs: + params.append(in_var_name) + + parameters = [parent_block.var(name) for name in params] + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + + inlinks = [parent_block.var(i.name) for i in self.inputs] + outlinks = self.outputs + + boot_memories = [] + pre_memories = [] + memories = [] + for _, mem in self.memories.iteritems(): + boot_memories.append(mem.init) + pre_memories.append(mem.pre_mem.name) + mem_var = rnn_block.var(mem.mem.name) + assert isinstance(mem_var, Variable) + new_mem = self.helper.create_tmp_variable(dtype=mem_var.data_type) + + rnn_block.append_op( + type='rnn_memory_helper', + inputs={'X': [mem_var]}, + outputs={'Out': [new_mem]}, + attrs={'data_type': mem_var.data_type}) + + memories.append(new_mem.name) + + parent_block.append_op( + type='recurrent', + inputs={ + 'inputs': inlinks, + 'initial_states': boot_memories, + 'parameters': parameters + }, + outputs={'outputs': outlinks, + 'step_scopes': [step_scope]}, + attrs={ + 'ex_states': pre_memories, + 'states': memories, + 'step_block': rnn_block + }) + + +class WhileGuard(BlockGuard): + def __init__(self, while_op): + if not isinstance(while_op, While): + raise TypeError("WhileGuard takes a while op") + super(WhileGuard, self).__init__(while_op.helper.main_program) + self.while_op = while_op + + def __enter__(self): + self.while_op.status = While.IN_WHILE_BLOCK + return super(WhileGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + return False + self.while_op.status = While.AFTER_WHILE_BLOCK + self.while_op.complete() + return super(WhileGuard, self).__exit__(exc_type, exc_val, exc_tb) + + +class While(object): + BEFORE_WHILE_BLOCK = 0 + IN_WHILE_BLOCK = 1 + AFTER_WHILE_BLOCK = 2 + + def __init__(self, cond, name=None, main_program=None): + self.helper = LayerHelper("while", name=name, main_program=main_program) + self.status = While.BEFORE_WHILE_BLOCK + if not isinstance(cond, Variable): + raise TypeError("condition should be a variable") + assert isinstance(cond, Variable) + if cond.data_type != core.DataType.BOOL: + raise TypeError("condition should be a bool variable") + if reduce(lambda a, b: a * b, cond.shape, 1) != 1: + raise TypeError("condition should be a bool scalar") + self.cond_var = cond + + def block(self): + return WhileGuard(self) + + def complete(self): + main_program = self.helper.main_program + while_block = main_program.current_block() + parent_block = main_program.block(main_program.current_block() + .parent_idx) + + inner_outputs = {self.cond_var.name} + x_name_list = set() + for op in while_block.ops: + for iname in op.input_names: + for in_var_name in op.input(iname): + if in_var_name not in inner_outputs: + x_name_list.add(in_var_name) + + for oname in op.output_names: + for out_var_name in op.output(oname): + inner_outputs.add(out_var_name) + + out_vars = [] + for inner_out_name in inner_outputs: + if inner_out_name in parent_block.vars: + out_vars.append(parent_block.var(inner_out_name)) + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + + parent_block.append_op( + type='while', + inputs={ + 'X': [parent_block.var(x_name) for x_name in x_name_list], + 'Condition': [self.cond_var] + }, + outputs={'Out': out_vars, + 'StepScopes': [step_scope]}, + attrs={'step_block': while_block}) + + +def lstm(x, + c_pre_init, + hidden_dim, + forget_bias=None, + main_program=None, + startup_program=None): + """ + This function helps create an operator for the LSTM (Long Short Term + Memory) cell that can be used inside an RNN. + """ + helper = LayerHelper('lstm_unit', **locals()) + rnn = StaticRNN() + with rnn.step(): + c_pre = rnn.memory(init=c_pre_init) + x_t = rnn.step_input(x) + + before_fc = concat( + input=[x_t, c_pre], + axis=1, + main_program=main_program, + startup_program=startup_program) + after_fc = fc(input=before_fc, + size=hidden_dim * 4, + main_program=main_program, + startup_program=startup_program) + + data_type = x.data_type + c = helper.create_tmp_variable(data_type) + h = helper.create_tmp_variable(data_type) + + helper.append_op( + type='lstm_unit', + inputs={"X": after_fc, + "C_prev": c_pre}, + outputs={"C": c, + "H": h}, + attrs={"forget_bias": forget_bias}) + + rnn.update_memory(c_pre, c) + rnn.output(h) + + return rnn() + + +def lod_rank_table(x, level=0, main_program=None): + """ + This function creates an operator for creating a LOD_RANK_TABLE + using the input x. + """ + helper = LayerHelper("lod_rank_table", **locals()) + table = helper.create_variable( + type=core.VarDesc.VarType.LOD_RANK_TABLE, + name=unique_name("lod_rank_table")) + helper.append_op( + type='lod_rank_table', + inputs={'X': x}, + outputs={'Out': table}, + attrs={'level': level}) + return table + + +def lod_tensor_to_array(x, table, main_program=None): + """ + This function creates an operator to convert an LOD_Tensor to + an array. + """ + helper = LayerHelper("lod_tensor_to_array", **locals()) + array = helper.create_variable( + name=unique_name("lod_tensor_to_array"), + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + dtype=x.data_type) + helper.append_op( + type='lod_tensor_to_array', + inputs={'X': x, + 'RankTable': table}, + outputs={'Out': array}) + return array + + +def array_to_lod_tensor(x, table, main_program=None): + """ + This function creates an operator to convert an array to a + LOD_Tensor. + """ + helper = LayerHelper("array_to_lod_tensor", **locals()) + tmp = helper.create_tmp_variable(dtype=x.data_type) + helper.append_op( + type="array_to_lod_tensor", + inputs={'X': x, + 'RankTable': table}, + outputs={'Out': tmp}) + return tmp + + +def fill_constant(shape, dtype, value, main_program=None): + """ + This function creates a tensor , with shape as mentioned in the input and + specified data_type and fills this up with a constant value that + comes in the input. It also sets the stop_gradient to be True. + """ + helper = LayerHelper("fill_constant", **locals()) + out = helper.create_tmp_variable(dtype=dtype) + helper.append_op( + type='fill_constant', + inputs={}, + outputs={'Out': [out]}, + attrs={ + 'shape': shape, + 'data_type': out.data_type, + 'value': float(value) + }) + out.stop_gradient = True + return out + + +def ones(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 1.0. + """ + return fill_constant(value=1.0, **locals()) + + +def zeros(shape, dtype, main_program=None): + """ + This function performs the same function as fill_constant() declared above + with the constant value being 0.0. + """ + return fill_constant(value=0.0, **locals()) + + +def increment(x, value=1.0, in_place=True, main_program=None): + """ + This function creates an operator to increment each value in the input + `x` by an amount: `value` as mentioned in the input parameter. This + operation is performed in-place by default. + """ + helper = LayerHelper("increment", **locals()) + if not in_place: + out = helper.create_tmp_variable(dtype=x.data_type) + else: + out = x + helper.append_op( + type='increment', + inputs={'X': [x]}, + outputs={'Out': [out]}, + attrs={'step': value}) + return out + + +def array_write(x, i, array=None, main_program=None): + """ + This function creates an operator to write the data out as a + LOD_TENSOR_ARRAY. + """ + helper = LayerHelper('array_write', **locals()) + if array is None: + array = helper.create_variable( + name="{0}.out".format(helper.name), + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + dtype=x.data_type) + helper.append_op( + type='write_to_array', + inputs={'X': [x], + 'I': [i]}, + outputs={'Out': [array]}) + return array + + +def create_array(dtype, main_program=None): + helper = LayerHelper("array", **locals()) + return helper.create_variable( + name="{0}.out".format(helper.name), + type=core.VarDesc.VarType.LOD_TENSOR_ARRAY, + dtype=dtype) + + +def less_than(x, y, cond=None, main_program=None): + helper = LayerHelper("less_than", **locals()) + if cond is None: + cond = helper.create_tmp_variable(dtype='bool') + cond.stop_gradient = True + + helper.append_op( + type='less_than', inputs={'X': [x], + 'Y': [y]}, outputs={'Out': [cond]}) + return cond + + +def array_read(array, i, main_program=None): + """ + This function creates an operator to read the data in as a + LOD_TENSOR_ARRAY. + """ + helper = LayerHelper('array_read', **locals()) + if not isinstance( + array, + Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY: + raise TypeError("array should be tensor array vairable") + out = helper.create_tmp_variable(dtype=array.data_type) + helper.append_op( + type='read_from_array', + inputs={'X': [array], + 'I': [i]}, + outputs={'Out': [out]}) + return out + + +def shrink_memory(x, i, table, main_program=None): + """ + This function creates an operator to shrink_rnn_memory using the RankTable + as mentioned in the input parameter. + """ + helper = LayerHelper('shrink_memory', **locals()) + out = helper.create_tmp_variable(dtype=x.data_type) + helper.append_op( + type='shrink_rnn_memory', + inputs={'X': [x], + 'I': [i], + 'RankTable': [table]}, + outputs={'Out': [out]}, + attrs={}) + return out + + +def array_length(array, main_program=None): + """ + This function creates an operator to find the length of the + LOD_TENSOR_ARRAY. + """ + helper = LayerHelper('array_length', **locals()) + tmp = helper.create_tmp_variable(dtype='int64') + tmp.stop_gradient = True + helper.append_op( + type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]}) + return tmp + + +class ConditionalBlockGuard(BlockGuard): + def __init__(self, block): + if not isinstance(block, ConditionalBlock): + raise TypeError("block should be conditional block") + super(ConditionalBlockGuard, self).__init__(block.helper.main_program) + self.block = block + + def __enter__(self): + return super(ConditionalBlockGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.block.complete() + return super(ConditionalBlockGuard, self).__exit__(exc_type, exc_val, + exc_tb) + + +class ConditionalBlock(object): + def __init__(self, inputs, name=None, main_program=None): + for each_input in inputs: + if not isinstance(each_input, Variable): + raise TypeError("Each input should be variable") + self.inputs = inputs + self.helper = LayerHelper( + 'conditional_block', name=name, main_program=main_program) + + def block(self): + return ConditionalBlockGuard(self) + + def complete(self): + inside_block = self.helper.main_program.current_block() + parent_block = self.helper.main_program.block(inside_block.parent_idx) + + intermediate = set() + params = set() + + for each_op in inside_block.ops: + assert isinstance(each_op, Operator) + for iname in each_op.input_names: + for in_var_name in each_op.input(iname): + if in_var_name not in intermediate: + params.add(in_var_name) + + for oname in each_op.output_names: + for out_var_name in each_op.output(oname): + intermediate.add(out_var_name) + input_set = set([ipt.name for ipt in self.inputs]) + + param_list = [ + parent_block.var(each_name) for each_name in params + if each_name not in input_set + ] + + out_list = [ + parent_block.var(var_name) for var_name in parent_block.vars + if var_name not in intermediate + ] + + step_scope = parent_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) + parent_block.append_op( + type='conditional_block', + inputs={ + 'X': self.inputs, + 'Params': param_list, + }, + outputs={'Out': out_list, + 'Scope': [step_scope]}, + attrs={'block': inside_block}) diff --git a/python/paddle/v2/framework/net_drawer.py b/python/paddle/v2/framework/net_drawer.py new file mode 100644 index 0000000000000000000000000000000000000000..045e267c253e2485e75df3fb95cc0e591ee29ea5 --- /dev/null +++ b/python/paddle/v2/framework/net_drawer.py @@ -0,0 +1,109 @@ +import argparse +import json +import logging +from collections import defaultdict + +import paddle.v2.framework.core as core +import paddle.v2.framework.proto.framework_pb2 as framework_pb2 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +try: + from graphviz import Digraph +except ImportError: + logger.info( + 'Cannot import graphviz, which is required for drawing a network. This ' + 'can usually be installed in python with "pip install graphviz". Also, ' + 'pydot requires graphviz to convert dot files to pdf: in ubuntu, this ' + 'can usually be installed with "sudo apt-get install graphviz".') + print('net_drawer will not run correctly. Please install the correct ' + 'dependencies.') + exit(0) + +OP_STYLE = { + 'shape': 'oval', + 'color': '#0F9D58', + 'style': 'filled', + 'fontcolor': '#FFFFFF' +} + +VAR_STYLE = {} + +GRAPH_STYLE = {"rankdir": "TB", } + +GRAPH_ID = 0 + + +def unique_id(): + def generator(): + GRAPH_ID += 1 + return GRAPH_ID + + return generator + + +def draw_node(op): + node = OP_STYLE + node["name"] = op.type + node["label"] = op.type + return node + + +def draw_edge(var_parent, op, var, arg): + edge = VAR_STYLE + edge["label"] = "%s(%s)" % (var.parameter, arg) + edge["head_name"] = op.type + edge["tail_name"] = var_parent[arg] + return edge + + +def parse_graph(program, graph, var_dict, **kwargs): + + # fill the known variables + for block in program.blocks: + for var in block.vars: + if not var_dict.has_key(var): + var_dict[var] = "Feed" + + proto = framework_pb2.ProgramDesc.FromString( + program.desc.serialize_to_string()) + for block in proto.blocks: + for op in block.ops: + graph.node(**draw_node(op)) + for o in op.outputs: + for arg in o.arguments: + var_dict[arg] = op.type + for e in op.inputs: + for arg in e.arguments: + if var_dict.has_key(arg): + graph.edge(**draw_edge(var_dict, op, e, arg)) + + +def draw_graph(startup_program, main_program, **kwargs): + if kwargs.has_key("graph_attr"): + GRAPH_STYLE.update(kwargs[graph_attr]) + if kwargs.has_key("node_attr"): + OP_STYLE.update(kwargs[node_attr]) + if kwargs.has_key("edge_attr"): + VAR_STYLE.update(kwargs[edge_attr]) + + graph_id = unique_id() + filename = kwargs.get("filename") + if filename == None: + filename = str(graph_id) + ".gv" + g = Digraph( + name=str(graph_id), + filename=filename, + graph_attr=GRAPH_STYLE, + node_attr=OP_STYLE, + edge_attr=VAR_STYLE, + **kwargs) + + var_dict = {} + parse_graph(startup_program, g, var_dict) + parse_graph(main_program, g, var_dict) + + if filename != None: + g.save() + return g diff --git a/python/paddle/v2/framework/nets.py b/python/paddle/v2/framework/nets.py index 8191b5ef44de840c90087ff4b9eb6923b3f1fbae..725d2fa7f5e7a862eea0ef9172a9e63858ebd0dd 100644 --- a/python/paddle/v2/framework/nets.py +++ b/python/paddle/v2/framework/nets.py @@ -10,23 +10,23 @@ def simple_img_conv_pool(input, pool_stride, act, pool_type='max', - program=None, - init_program=None): + main_program=None, + startup_program=None): conv_out = layers.conv2d( input=input, num_filters=num_filters, filter_size=filter_size, act=act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) pool_out = layers.pool2d( input=conv_out, pool_size=pool_size, pool_type=pool_type, pool_stride=pool_stride, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return pool_out @@ -40,14 +40,14 @@ def img_conv_group(input, conv_batchnorm_drop_rate=None, pool_stride=1, pool_type=None, - program=None, - init_program=None): + main_program=None, + startup_program=None): """ Image Convolution Group, Used for vgg net. """ tmp = input assert isinstance(conv_num_filter, list) or \ - isinstance(conv_num_filter, tuple) + isinstance(conv_num_filter, tuple) def __extend_list__(obj): if not hasattr(obj, '__len__'): @@ -71,49 +71,51 @@ def img_conv_group(input, filter_size=conv_filter_size[i], padding=conv_padding[i], act=local_conv_act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) if conv_with_batchnorm[i]: tmp = layers.batch_norm( input=tmp, act=conv_act, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) drop_rate = conv_batchnorm_drop_rate[i] if abs(drop_rate) > 1e-5: tmp = layers.dropout( x=tmp, dropout_prob=drop_rate, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) pool_out = layers.pool2d( input=tmp, pool_size=pool_size, pool_type=pool_type, pool_stride=pool_stride, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return pool_out def sequence_conv_pool(input, num_filters, filter_size, + act="sigmoid", pool_type="max", - program=None, - init_program=None): + main_program=None, + startup_program=None): conv_out = layers.sequence_conv( input=input, num_filters=num_filters, filter_size=filter_size, - program=program, - init_program=init_program) + act=act, + main_program=main_program, + startup_program=startup_program) pool_out = layers.sequence_pool( input=conv_out, pool_type=pool_type, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return pool_out diff --git a/python/paddle/v2/framework/optimizer.py b/python/paddle/v2/framework/optimizer.py index 4c608f96bdf0ca715fc89c0752e891f8c2b80d87..f06c0fb98d572fb54a85996668cc6f32726ec9de 100644 --- a/python/paddle/v2/framework/optimizer.py +++ b/python/paddle/v2/framework/optimizer.py @@ -1,8 +1,11 @@ from collections import defaultdict import paddle.v2.framework.framework as framework +from paddle.v2.framework.framework import unique_name, Program from paddle.v2.framework.backward import append_backward_ops +from paddle.v2.framework.initializer import ConstantInitializer from paddle.v2.framework.regularizer import append_regularization_ops +from paddle.v2.framework.layer_helper import LayerHelper __all__ = [ 'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer', @@ -25,21 +28,28 @@ class Optimizer(object): # to train. These variables are called accumulators. # {accum_name : { paramter_name : accumulator_for_parameter, ...}, ...} self._accumulators = defaultdict(lambda: dict()) + self.helper = None def _append_optimize_op(self, block, param_and_grad): """ append optimize operator to block and return all the added optimize_op """ raise NotImplementedError() - def _initialize_tensors(self, block): - """Create all necessary tensors, that will be shared for all parameter updates. - - Tensors like learning rate should be initialized here. - - Args: - block: the block in which the loss variable is present - """ - pass + def _create_param_lr(self, param_and_grad): + # create learning rate variable for every parameter + param = param_and_grad[0] + param_lr = param.optimize_attr['learning_rate'] + param_lr_shape = [1] + param_lr_var = self.helper.create_global_variable( + name=unique_name("learning_rate"), + dtype='float32', + shape=param_lr_shape, + lod_level=1, + persistable=True) + param_lr = param_lr * self._learning_rate + self.helper.set_variable_initializer( + var=param_lr_var, initializer=ConstantInitializer(param_lr)) + return param_lr_var def _create_accumulators(self, block, parameters): """Create all accumulators needed by the parameters @@ -63,7 +73,7 @@ class Optimizer(object): """ pass - def _add_accumulator(self, block, name, param, dtype=None, fill_value=0.0): + def _add_accumulator(self, name, param, dtype=None, fill_value=0.0): """Utility function to add an accumulator for a parameter Args: @@ -77,22 +87,17 @@ class Optimizer(object): param.name in self._accumulators[name]): raise Exception("Accumulator {} already exists for parmeter {}". format(name, param.name)) - global_block = block.program.global_block() - param_shape = list(param.shape) - param_acc = global_block.create_var( - dtype=dtype, shape=param_shape, lod_level=0) - - # Initialize the accumulator with fill_value - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": param_acc}, - attrs={"shape": param_shape, - "value": fill_value}) - - # Add to accumulators dict - self._accumulators[name][param.name] = param_acc + + assert isinstance(self.helper, LayerHelper) + var = self.helper.create_global_variable( + name=unique_name(name), + persistable=True, + dtype=dtype or param.data_type, + type=param.type, + shape=param.shape) + self.helper.set_variable_initializer( + var, initializer=ConstantInitializer(value=float(fill_value))) + self._accumulators[name][param.name] = var def _get_accumulator(self, name, param): """Utility function to fetch an accumulator for a parameter @@ -130,7 +135,10 @@ class Optimizer(object): return increment_op - def create_optimization_pass(self, parameters_and_grads, loss): + def create_optimization_pass(self, + parameters_and_grads, + loss, + startup_program=None): """Add optimization operators to update gradients to variables. Args: @@ -142,6 +150,7 @@ class Optimizer(object): optimization. This will include parameter update ops, global step update ops and any other custom ops required by subclasses to manage their internal state. + :param startup_program: """ # This is a default implementation of create_optimization_pass that # can be shared by most optimizers. This implementation assumes that @@ -151,10 +160,13 @@ class Optimizer(object): # for parameters and extend _finish_update method to add custom ops. # Create any accumulators + program = loss.block.program + self.helper = LayerHelper( + self.__class__.__name__, + main_program=program, + startup_program=startup_program) self._create_accumulators(loss.block, [p[0] for p in parameters_and_grads]) - # Create any necessary tensors - self._initialize_tensors(loss.block) optimize_ops = [] for param_and_grad in parameters_and_grads: @@ -177,7 +189,11 @@ class Optimizer(object): return_ops.append(self._increment_global_step(loss.block)) return return_ops - def minimize(self, loss, parameter_list=None, no_grad_set=None): + def minimize(self, + loss, + startup_program=None, + parameter_list=None, + no_grad_set=None): """Add operations to minimize `loss` by updating `parameter_list`. This method combines interface `append_backward_ops()` and @@ -187,7 +203,8 @@ class Optimizer(object): set()) # Add regularization if any params_grads = append_regularization_ops(params_grads) - optimize_ops = self.create_optimization_pass(params_grads, loss) + optimize_ops = self.create_optimization_pass(params_grads, loss, + startup_program) return optimize_ops @@ -201,22 +218,6 @@ class SGDOptimizer(Optimizer): self.type = "sgd" self._learning_rate = learning_rate - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) - def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -226,7 +227,7 @@ class SGDOptimizer(Optimizer): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={"ParamOut": param_and_grad[0]}) @@ -251,27 +252,11 @@ class MomentumOptimizer(Optimizer): self._momentum = momentum self._use_nesterov = bool(use_nesterov) - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(block, self._velocity_acc_str, p, 'float32') + self._add_accumulator(self._velocity_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -285,14 +270,14 @@ class MomentumOptimizer(Optimizer): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Velocity": velocity_acc, - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={ "ParamOut": param_and_grad[0], "VelocityOut": velocity_acc }, attrs={"mu": self._momentum, - "useNesterov": self._use_nesterov}) + "use_nesterov": self._use_nesterov}) return momentum_op @@ -310,27 +295,11 @@ class AdagradOptimizer(Optimizer): self._learning_rate = learning_rate self._epsilon = epsilon - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) for p in parameters: - self._add_accumulator(block, self._moment_acc_str, p, 'float32') + self._add_accumulator(self._moment_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -345,7 +314,7 @@ class AdagradOptimizer(Optimizer): "Param": param_and_grad[0], "Grad": param_and_grad[1], "Moment": moment_acc, - "LearningRate": self._lr + "LearningRate": self._create_param_lr(param_and_grad) }, outputs={"ParamOut": param_and_grad[0], "MomentOut": moment_acc}, @@ -377,52 +346,35 @@ class AdamOptimizer(Optimizer): self._beta2 = beta2 self._epsilon = epsilon - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) - def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) - global_block = block.program.global_block() + main_block = block.program.global_block() # Create beta1 and beta2 power tensors beta_shape = [1] - # Create variables for beta1 and beta2 powers - self._beta1_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - self._beta2_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - - # Initialize beta1 and beta2 power accumulators - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta1_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta1}) - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta2_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta2}) + self._beta1_pow_acc = self.helper.create_global_variable( + name=unique_name('beta1_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + self.helper.set_variable_initializer( + self._beta1_pow_acc, initializer=ConstantInitializer(self._beta1)) + + self._beta2_pow_acc = self.helper.create_global_variable( + name=unique_name('beta2_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + + self.helper.set_variable_initializer( + self._beta2_pow_acc, initializer=ConstantInitializer(self._beta2)) # Create accumulator tensors for first and second moments for p in parameters: - self._add_accumulator(block, self._moment1_acc_str, p, 'float32') - self._add_accumulator(block, self._moment2_acc_str, p, 'float32') + self._add_accumulator(self._moment1_acc_str, p) + self._add_accumulator(self._moment2_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -437,7 +389,7 @@ class AdamOptimizer(Optimizer): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr, + "LearningRate": self._create_param_lr(param_and_grad), "Moment1": moment1, "Moment2": moment2, "Beta1Pow": self._beta1_pow_acc, @@ -460,14 +412,14 @@ class AdamOptimizer(Optimizer): """Update Beta1 and Beta2 Power accumulators """ assert isinstance(block, framework.Block) - global_block = block.program.global_block() - scale_beta1 = global_block.append_op( + main_block = block.program.global_block() + scale_beta1 = main_block.append_op( type="scale", inputs={"X": self._beta1_pow_acc}, outputs={"Out": self._beta1_pow_acc}, attrs={"scale": self._beta1}) - scale_beta2 = global_block.append_op( + scale_beta2 = main_block.append_op( type="scale", inputs={"X": self._beta2_pow_acc}, outputs={"Out": self._beta2_pow_acc}, @@ -499,44 +451,22 @@ class AdamaxOptimizer(Optimizer): self._beta2 = beta2 self._epsilon = epsilon - def _initialize_tensors(self, block): - assert isinstance(block, framework.Block) - lr_shape = [1] - # create a variable for learning_rate - self._lr = block.create_var( - dtype="float32", shape=lr_shape, lod_level=0) - - # create an op to init the learning_rate - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - block.append_op( - type="fill_constant", - outputs={"Out": self._lr}, - attrs={"shape": lr_shape, - "value": self._learning_rate}) - def _create_accumulators(self, block, parameters): - assert isinstance(block, framework.Block) - - global_block = block.program.global_block() # Create beta1 power accumulator tensor beta_shape = [1] - self._beta1_pow_acc = global_block.create_var( - dtype="float32", shape=beta_shape, lod_level=0) - - # Initialize beta1 power accumulator - # FIXME: Fix when Initialization design has been implemented - # https://github.com/PaddlePaddle/Paddle/pull/4852 - global_block.append_op( - type="fill_constant", - outputs={"Out": self._beta1_pow_acc}, - attrs={"shape": beta_shape, - "value": self._beta1}) + self._beta1_pow_acc = self.helper.create_global_variable( + name=unique_name('beta1_pow_acc'), + dtype='float32', + shape=beta_shape, + lod_level=0, + persistable=True) + self.helper.set_variable_initializer( + self._beta1_pow_acc, initializer=ConstantInitializer(self._beta1)) # Create accumulator tensors for first moment and infinity norm for p in parameters: - self._add_accumulator(block, self._moment_acc_str, p, 'float32') - self._add_accumulator(block, self._inf_norm_acc_str, p, 'float32') + self._add_accumulator(self._moment_acc_str, p) + self._add_accumulator(self._inf_norm_acc_str, p) def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) @@ -550,7 +480,7 @@ class AdamaxOptimizer(Optimizer): inputs={ "Param": param_and_grad[0], "Grad": param_and_grad[1], - "LearningRate": self._lr, + "LearningRate": self._create_param_lr(param_and_grad), "Moment": moment, "InfNorm": inf_norm, "Beta1Pow": self._beta1_pow_acc @@ -572,8 +502,8 @@ class AdamaxOptimizer(Optimizer): """Update Beta1 Power accumulator """ assert isinstance(block, framework.Block) - global_block = block.program.global_block() - scale_beta1 = global_block.append_op( + main_block = block.program.global_block() + scale_beta1 = main_block.append_op( type="scale", inputs={"X": self._beta1_pow_acc}, outputs={"Out": self._beta1_pow_acc}, diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 4d7664469e481344cf9eea84688f068b4fb99dee..e795627bfe9e8ad0c196349a332e62e975f20aa3 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -3,3 +3,5 @@ string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") foreach(src ${TEST_OPS}) py_test(${src} SRCS ${src}.py) endforeach() + +add_subdirectory(book) diff --git a/python/paddle/v2/framework/tests/book/CMakeLists.txt b/python/paddle/v2/framework/tests/book/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d7664469e481344cf9eea84688f068b4fb99dee --- /dev/null +++ b/python/paddle/v2/framework/tests/book/CMakeLists.txt @@ -0,0 +1,5 @@ +file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") +string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") +foreach(src ${TEST_OPS}) + py_test(${src} SRCS ${src}.py) +endforeach() diff --git a/python/paddle/v2/framework/tests/test_fit_a_line.py b/python/paddle/v2/framework/tests/book/test_fit_a_line.py similarity index 65% rename from python/paddle/v2/framework/tests/test_fit_a_line.py rename to python/paddle/v2/framework/tests/book/test_fit_a_line.py index 7c2ef61fe103655369fd6fe68733e810d4e19d1d..6e09b88dca34de2579131e7bdc16b26cf6cde49c 100644 --- a/python/paddle/v2/framework/tests/test_fit_a_line.py +++ b/python/paddle/v2/framework/tests/book/test_fit_a_line.py @@ -3,40 +3,44 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.io import save_persistables, load_persistables from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() x = layers.data( name='x', shape=[13], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) y_predict = layers.fc(input=x, size=1, act=None, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) y = layers.data( name='y', shape=[1], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.square_error_cost( - input=y_predict, label=y, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) + input=y_predict, + label=y, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +opts = sgd_optimizer.minimize(avg_cost, startup_program) BATCH_SIZE = 20 @@ -48,12 +52,12 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): - save_persistables(exe, "./fit_a_line.model/", program=program) - load_persistables(exe, "./fit_a_line.model/", program=program) + save_persistables(exe, "./fit_a_line.model/", main_program=main_program) + load_persistables(exe, "./fit_a_line.model/", main_program=main_program) for data in train_reader(): x_data = np.array(map(lambda x: x[0], data)).astype("float32") y_data = np.array(map(lambda x: x[1], data)).astype("float32") @@ -65,7 +69,7 @@ for pass_id in range(PASS_NUM): tensor_y = core.LoDTensor() tensor_y.set(y_data, place) # print tensor_y.get_dims() - outs = exe.run(program, + outs = exe.run(main_program, feed={'x': tensor_x, 'y': tensor_y}, fetch_list=[avg_cost]) diff --git a/python/paddle/v2/framework/tests/test_image_classification_train.py b/python/paddle/v2/framework/tests/book/test_image_classification_train.py similarity index 53% rename from python/paddle/v2/framework/tests/test_image_classification_train.py rename to python/paddle/v2/framework/tests/book/test_image_classification_train.py index 6b6dec4976d510fae7e987ad0276b049bbcb96fa..a4165da9703c55ae3347123409407f0cae30856f 100644 --- a/python/paddle/v2/framework/tests/test_image_classification_train.py +++ b/python/paddle/v2/framework/tests/book/test_image_classification_train.py @@ -1,24 +1,23 @@ +import numpy as np import paddle.v2 as paddle +import paddle.v2.framework.core as core import paddle.v2.framework.layers as layers import paddle.v2.framework.nets as nets -import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer - -from paddle.v2.framework.framework import Program, g_program from paddle.v2.framework.executor import Executor - -import numpy as np +from paddle.v2.framework.framework import g_startup_program, g_main_program +from paddle.v2.framework.initializer import XavierInitializer -def resnet_cifar10(input, depth=32, program=None, init_program=None): +def resnet_cifar10(input, depth=32, main_program=None, startup_program=None): def conv_bn_layer(input, ch_out, filter_size, stride, padding, act='relu', - program=None, - init_program=None): + main_program=None, + startup_program=None): tmp = layers.conv2d( input=input, filter_size=filter_size, @@ -27,10 +26,13 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): padding=padding, act=None, bias_attr=False, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return layers.batch_norm( - input=tmp, act=act, program=program, init_program=init_program) + input=tmp, + act=act, + main_program=main_program, + startup_program=startup_program) def shortcut(input, ch_in, ch_out, stride, program, init_program): if ch_in != ch_out: @@ -43,16 +45,16 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): ch_in, ch_out, stride, - program=program, - init_program=init_program): + main_program=main_program, + startup_program=startup_program): tmp = conv_bn_layer( input, ch_out, 3, stride, 1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) tmp = conv_bn_layer( tmp, ch_out, @@ -60,21 +62,22 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): 1, 1, act=None, - program=program, - init_program=init_program) - short = shortcut(input, ch_in, ch_out, stride, program, init_program) + main_program=main_program, + startup_program=startup_program) + short = shortcut(input, ch_in, ch_out, stride, main_program, + startup_program) return layers.elementwise_add( x=tmp, y=short, act='relu', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) def layer_warp(block_func, input, ch_in, ch_out, count, stride, program, - init_program): - tmp = block_func(input, ch_in, ch_out, stride, program, init_program) + startup_program): + tmp = block_func(input, ch_in, ch_out, stride, program, startup_program) for i in range(1, count): - tmp = block_func(tmp, ch_out, ch_out, 1, program, init_program) + tmp = block_func(tmp, ch_out, ch_out, 1, program, startup_program) return tmp assert (depth - 2) % 6 == 0 @@ -85,8 +88,8 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): filter_size=3, stride=1, padding=1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) res1 = layer_warp( basicblock, conv1, @@ -94,8 +97,8 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): 16, n, 1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) res2 = layer_warp( basicblock, res1, @@ -103,8 +106,8 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): 32, n, 2, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) res3 = layer_warp( basicblock, res2, @@ -112,25 +115,25 @@ def resnet_cifar10(input, depth=32, program=None, init_program=None): 64, n, 2, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) pool = layers.pool2d( input=res3, pool_size=8, pool_type='avg', pool_stride=1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) return pool -def vgg16_bn_drop(input, program, init_program): +def vgg16_bn_drop(input, main_program=None, startup_program=None): def conv_block(input, num_filter, groups, dropouts, - program=None, - init_program=None): + main_program=None, + startup_program=None): return nets.img_conv_group( input=input, pool_size=2, @@ -141,74 +144,75 @@ def vgg16_bn_drop(input, program, init_program): conv_with_batchnorm=True, conv_batchnorm_drop_rate=dropouts, pool_type='max', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) - conv1 = conv_block(input, 64, 2, [0.3, 0], program, init_program) - conv2 = conv_block(conv1, 128, 2, [0.4, 0], program, init_program) - conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0], program, init_program) - conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0], program, init_program) - conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0], program, init_program) + conv1 = conv_block(input, 64, 2, [0.3, 0], main_program, startup_program) + conv2 = conv_block(conv1, 128, 2, [0.4, 0], main_program, startup_program) + conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0], main_program, + startup_program) + conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0], main_program, + startup_program) + conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0], main_program, + startup_program) drop = layers.dropout( - x=conv5, dropout_prob=0.5, program=program, init_program=init_program) + x=conv5, + dropout_prob=0.5, + main_program=main_program, + startup_program=startup_program) fc1 = layers.fc(input=drop, size=512, act=None, - program=program, - init_program=init_program) + param_attr={"initializer": XavierInitializer()}, + main_program=main_program, + startup_program=startup_program) reshape1 = layers.reshape( x=fc1, shape=list(fc1.shape + (1, 1)), - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) bn = layers.batch_norm( - input=reshape1, act='relu', program=program, init_program=init_program) + input=reshape1, + act='relu', + main_program=main_program, + startup_program=startup_program) drop2 = layers.dropout( - x=bn, dropout_prob=0.5, program=program, init_program=init_program) + x=bn, + dropout_prob=0.5, + main_program=main_program, + startup_program=startup_program) fc2 = layers.fc(input=drop2, size=512, act=None, - program=program, - init_program=init_program) + param_attr={"initializer": XavierInitializer()}, + main_program=main_program, + startup_program=startup_program) return fc2 -init_program = Program() -program = Program() - classdim = 10 data_shape = [3, 32, 32] -images = layers.data( - name='pixel', shape=data_shape, data_type='float32', program=program) - -label = layers.data( - name='label', - shape=[1], - data_type='int64', - program=program, - init_program=init_program) +images = layers.data(name='pixel', shape=data_shape, data_type='float32') +label = layers.data(name='label', shape=[1], data_type='int64') # Add neural network config # option 1. resnet -net = resnet_cifar10(images, 32, program, init_program) +# net = resnet_cifar10(images, 32) # option 2. vgg -# net = vgg16_bn_drop(images, program, init_program) +net = vgg16_bn_drop(images) # print(program) -predict = layers.fc(input=net, - size=classdim, - act='softmax', - program=program, - init_program=init_program) -cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) +predict = layers.fc(input=net, size=classdim, act='softmax') +cost = layers.cross_entropy(input=predict, label=label) +avg_cost = layers.mean(x=cost) +accuracy = layers.accuracy(input=predict, label=label) -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +# optimizer = optimizer.SGDOptimizer(learning_rate=0.001) +optimizer = optimizer.AdamOptimizer(learning_rate=0.001) +opts = optimizer.minimize(avg_cost) BATCH_SIZE = 128 PASS_NUM = 1 @@ -221,7 +225,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(g_startup_program, feed={}, fetch_list=[]) for pass_id in range(PASS_NUM): batch_id = 0 @@ -239,14 +243,15 @@ for pass_id in range(PASS_NUM): tensor_img.set(img_data, place) tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(g_main_program, feed={"pixel": tensor_img, "label": tensor_y}, - fetch_list=[avg_cost]) + fetch_list=[avg_cost, accuracy]) loss = np.array(outs[0]) + acc = np.array(outs[1]) print("pass_id:" + str(pass_id) + " batch_id:" + str(batch_id) + - " loss:" + str(loss)) + " loss:" + str(loss) + " acc:" + str(acc)) batch_id = batch_id + 1 if batch_id > 1: diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py b/python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py similarity index 65% rename from python/paddle/v2/framework/tests/test_recognize_digits_conv.py rename to python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py index 92b1d0542619b765cc32c98f59604cfc73d7d6d4..66c629eb4261a9b971f25611d8e49f0cb671304a 100644 --- a/python/paddle/v2/framework/tests/test_recognize_digits_conv.py +++ b/python/paddle/v2/framework/tests/book/test_recognize_digits_conv.py @@ -4,26 +4,26 @@ import paddle.v2.framework.nets as nets import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() images = layers.data( name='pixel', shape=[1, 28, 28], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) label = layers.data( name='label', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, @@ -31,8 +31,8 @@ conv_pool_1 = nets.simple_img_conv_pool( pool_size=2, pool_stride=2, act="relu", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) conv_pool_2 = nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -40,22 +40,30 @@ conv_pool_2 = nets.simple_img_conv_pool( pool_size=2, pool_stride=2, act="relu", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) predict = layers.fc(input=conv_pool_2, size=10, act="softmax", - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program) + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean(x=cost, main_program=main_program) accuracy = layers.accuracy( - input=predict, label=label, program=program, init_program=init_program) + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +# optimizer = optimizer.MomentumOptimizer(learning_rate=0.1 / 128.0, +# momentum=0.9) +optimizer = optimizer.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.999) +opts = optimizer.minimize(avg_cost, startup_program) BATCH_SIZE = 50 PASS_NUM = 3 @@ -67,7 +75,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) for pass_id in range(PASS_NUM): count = 0 @@ -82,7 +90,7 @@ for pass_id in range(PASS_NUM): tensor_img.set(img_data, place) tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={"pixel": tensor_img, "label": tensor_y}, fetch_list=[avg_cost, accuracy]) diff --git a/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py b/python/paddle/v2/framework/tests/book/test_recognize_digits_mlp.py similarity index 64% rename from python/paddle/v2/framework/tests/test_recognize_digits_mlp.py rename to python/paddle/v2/framework/tests/book/test_recognize_digits_mlp.py index 9916569d042d5d5077732281487ca3ae4366b39c..076cf882160cd53f45ef291d82ba57ada843a287 100644 --- a/python/paddle/v2/framework/tests/test_recognize_digits_mlp.py +++ b/python/paddle/v2/framework/tests/book/test_recognize_digits_mlp.py @@ -11,14 +11,14 @@ from paddle.v2.framework.initializer import UniformInitializer import numpy as np BATCH_SIZE = 128 -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() image = layers.data( name='x', shape=[784], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) param_attr = { 'name': None, @@ -30,36 +30,45 @@ param_attr = { hidden1 = layers.fc(input=image, size=128, act='relu', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) hidden2 = layers.fc(input=hidden1, size=64, act='relu', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) predict = layers.fc(input=hidden2, size=10, act='softmax', - program=program, - init_program=init_program, + main_program=main_program, + startup_program=startup_program, param_attr=param_attr) label = layers.data( name='y', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( - input=predict, label=label, program=program, init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) - -sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) +accuracy = layers.accuracy( + input=predict, + label=label, + main_program=main_program, + startup_program=startup_program) + +optimizer = optimizer.MomentumOptimizer(learning_rate=0.001, momentum=0.9) +opts = optimizer.minimize(avg_cost, startup_program) train_reader = paddle.batch( paddle.reader.shuffle( @@ -69,7 +78,7 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): @@ -84,11 +93,12 @@ for pass_id in range(PASS_NUM): tensor_y = core.LoDTensor() tensor_y.set(y_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={'x': tensor_x, 'y': tensor_y}, - fetch_list=[avg_cost]) + fetch_list=[avg_cost, accuracy]) out = np.array(outs[0]) + acc = np.array(outs[1]) if out[0] < 5.0: exit(0) # if avg cost less than 5.0, we think our code is good. exit(1) diff --git a/python/paddle/v2/framework/tests/book/test_recommender_system.py b/python/paddle/v2/framework/tests/book/test_recommender_system.py new file mode 100644 index 0000000000000000000000000000000000000000..31562b4391d16b831d53801cfa21c7bdf8c3ab8d --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_recommender_system.py @@ -0,0 +1,315 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor + +import numpy as np + +startup_program = Program() +main_program = Program() +is_sparse = True +use_gpu = False +BATCH_SIZE = 256 + + +def get_usr_combined_features(): + # FIXME(dzh) : old API integer_value(10) may has range check. + # currently we don't have user configurated check. + + USR_DICT_SIZE = paddle.dataset.movielens.max_user_id() + 1 + + uid = layers.data( + name='user_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + usr_emb = layers.embedding( + input=uid, + data_type='float32', + size=[USR_DICT_SIZE, 32], + param_attr={'name': 'user_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_fc = layers.fc(input=usr_emb, + size=32, + main_program=main_program, + startup_program=startup_program) + + USR_GENDER_DICT_SIZE = 2 + + usr_gender_id = layers.data( + name='gender_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + usr_gender_emb = layers.embedding( + input=usr_gender_id, + size=[USR_GENDER_DICT_SIZE, 16], + param_attr={'name': 'gender_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_gender_fc = layers.fc(input=usr_gender_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + USR_AGE_DICT_SIZE = len(paddle.dataset.movielens.age_table) + usr_age_id = layers.data( + name='age_id', + shape=[1], + data_type="int64", + main_program=main_program, + startup_program=startup_program) + + usr_age_emb = layers.embedding( + input=usr_age_id, + size=[USR_AGE_DICT_SIZE, 16], + is_sparse=is_sparse, + param_attr={'name': 'age_table'}, + main_program=main_program, + startup_program=startup_program) + + usr_age_fc = layers.fc(input=usr_age_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + USR_JOB_DICT_SIZE = paddle.dataset.movielens.max_job_id() + 1 + usr_job_id = layers.data( + name='job_id', + shape=[1], + data_type="int64", + main_program=main_program, + startup_program=startup_program) + + usr_job_emb = layers.embedding( + input=usr_job_id, + size=[USR_JOB_DICT_SIZE, 16], + param_attr={'name': 'job_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + usr_job_fc = layers.fc(input=usr_job_emb, + size=16, + main_program=main_program, + startup_program=startup_program) + + concat_embed = layers.concat( + input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc], + axis=1, + main_program=main_program, + startup_program=startup_program) + + usr_combined_features = layers.fc(input=concat_embed, + size=200, + act="tanh", + main_program=main_program, + startup_program=startup_program) + + return usr_combined_features + + +def get_mov_combined_features(): + + MOV_DICT_SIZE = paddle.dataset.movielens.max_movie_id() + 1 + + mov_id = layers.data( + name='movie_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_emb = layers.embedding( + input=mov_id, + data_type='float32', + size=[MOV_DICT_SIZE, 32], + param_attr={'name': 'movie_table'}, + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_fc = layers.fc(input=mov_emb, + size=32, + main_program=main_program, + startup_program=startup_program) + + CATEGORY_DICT_SIZE = len(paddle.dataset.movielens.movie_categories()) + + category_id = layers.data( + name='category_id', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_categories_emb = layers.embedding( + input=category_id, + size=[CATEGORY_DICT_SIZE, 32], + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_categories_hidden = layers.sequence_pool( + input=mov_categories_emb, + pool_type="sum", + main_program=main_program, + startup_program=startup_program) + + MOV_TITLE_DICT_SIZE = len(paddle.dataset.movielens.get_movie_title_dict()) + + mov_title_id = layers.data( + name='movie_title', + shape=[1], + data_type='int64', + main_program=main_program, + startup_program=startup_program) + + mov_title_emb = layers.embedding( + input=mov_title_id, + size=[MOV_TITLE_DICT_SIZE, 32], + is_sparse=is_sparse, + main_program=main_program, + startup_program=startup_program) + + mov_title_conv = nets.sequence_conv_pool( + input=mov_title_emb, + num_filters=32, + filter_size=3, + act="tanh", + pool_type="sum", + main_program=main_program, + startup_program=startup_program) + + concat_embed = layers.concat( + input=[mov_fc, mov_categories_hidden, mov_title_conv], + axis=1, + main_program=main_program, + startup_program=startup_program) + + # FIXME(dzh) : need tanh operator + mov_combined_features = layers.fc(input=concat_embed, + size=200, + act="tanh", + main_program=main_program, + startup_program=startup_program) + + return mov_combined_features + + +def model(): + usr_combined_features = get_usr_combined_features() + mov_combined_features = get_mov_combined_features() + + # need cos sim + inference = layers.cos_sim( + X=usr_combined_features, + Y=mov_combined_features, + main_program=main_program, + startup_program=startup_program) + + label = layers.data( + name='score', + shape=[1], + data_type='float32', + main_program=main_program, + startup_program=startup_program) + + square_cost = layers.square_error_cost( + input=inference, + label=label, + main_program=main_program, + startup_program=startup_program) + + avg_cost = layers.mean( + x=square_cost, + main_program=main_program, + startup_program=startup_program) + + return avg_cost + + +def main(): + cost = model() + sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.2) + opts = sgd_optimizer.minimize(cost, startup_program=startup_program) + block = main_program.block(0) + + if use_gpu: + place = core.GPUPlace(0) + else: + place = core.CPUPlace() + + exe = Executor(place) + exe.run(startup_program, feed={}, fetch_list=[]) + + train_reader = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.movielens.train(), buf_size=8192), + batch_size=BATCH_SIZE) + + feeding = { + 'user_id': 0, + 'gender_id': 1, + 'age_id': 2, + 'job_id': 3, + 'movie_id': 4, + 'category_id': 5, + 'movie_title': 6, + 'score': 7 + } + + def func_feed(feeding, data): + feed_tensors = {} + for (key, idx) in feeding.iteritems(): + tensor = core.LoDTensor() + if key != "category_id" and key != "movie_title": + if key == "score": + numpy_data = np.array(map(lambda x: x[idx], data)).astype( + "float32") + else: + numpy_data = np.array(map(lambda x: x[idx], data)).astype( + "int64") + else: + numpy_data = map(lambda x: np.array(x[idx]).astype("int64"), + data) + lod_info = [len(item) for item in numpy_data] + offset = 0 + lod = [offset] + for item in lod_info: + offset += item + lod.append(offset) + numpy_data = np.concatenate(numpy_data, axis=0) + tensor.set_lod([lod]) + + numpy_data = numpy_data.reshape([numpy_data.shape[0], 1]) + tensor.set(numpy_data, place) + feed_tensors[key] = tensor + return feed_tensors + + PASS_NUM = 100 + for pass_id in range(PASS_NUM): + for data in train_reader(): + outs = exe.run(main_program, + feed=func_feed(feeding, data), + fetch_list=[cost]) + out = np.array(outs[0]) + if out[0] < 6.0: + # if avg cost less than 6.0, we think our code is good. + exit(0) + + +main() diff --git a/python/paddle/v2/framework/tests/book/test_understand_sentiment_conv.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..eb377e9264b6031e9bf484a90b7c2b39442407f1 --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_understand_sentiment_conv.py @@ -0,0 +1,99 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program, g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def convolution_net(input_dim, class_dim=2, emb_dim=32, hid_dim=32): + data = layers.data(name="words", shape=[1], data_type="int64") + label = layers.data(name="label", shape=[1], data_type="int64") + + emb = layers.embedding(input=data, size=[input_dim, emb_dim]) + conv_3 = nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=3, + act="tanh", + pool_type="sqrt") + conv_4 = nets.sequence_conv_pool( + input=emb, + num_filters=hid_dim, + filter_size=4, + act="tanh", + pool_type="sqrt") + prediction = layers.fc(input=[conv_3, conv_4], + size=class_dim, + act="softmax") + cost = layers.cross_entropy(input=prediction, label=label) + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def main(): + BATCH_SIZE = 100 + PASS_NUM = 5 + + word_dict = paddle.dataset.imdb.word_dict() + dict_dim = len(word_dict) + class_dim = 2 + + cost, acc = convolution_net(input_dim=dict_dim, class_dim=class_dim) + + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000), + batch_size=BATCH_SIZE) + place = core.CPUPlace() + exe = Executor(place) + + exe.run(g_startup_program) + + for pass_id in xrange(PASS_NUM): + for data in train_data(): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([BATCH_SIZE, 1]) + + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if cost_val < 1.0 and acc_val > 0.7: + exit(0) + exit(1) + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py new file mode 100644 index 0000000000000000000000000000000000000000..2457c71e1a627c6d3edb298ab463a5d01243cea3 --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_understand_sentiment_dynamic_lstm.py @@ -0,0 +1,110 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.nets as nets +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import Program, g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def stacked_lstm_net(input_dim, + class_dim=2, + emb_dim=128, + hid_dim=512, + stacked_num=3): + assert stacked_num % 2 == 1 + data = layers.data(name="words", shape=[1], data_type="int64") + label = layers.data(name="label", shape=[1], data_type="int64") + + emb = layers.embedding(input=data, size=[input_dim, emb_dim]) + # add bias attr + + # TODO(qijun) linear act + fc1 = layers.fc(input=emb, size=hid_dim) + lstm1, cell1 = layers.dynamic_lstm(input=fc1, size=hid_dim) + + inputs = [fc1, lstm1] + + for i in range(2, stacked_num + 1): + fc = layers.fc(input=inputs, size=hid_dim) + lstm, cell = layers.dynamic_lstm( + input=fc, size=hid_dim, is_reverse=(i % 2) == 0) + inputs = [fc, lstm] + + fc_last = layers.sequence_pool(input=inputs[0], pool_type='max') + lstm_last = layers.sequence_pool(input=inputs[1], pool_type='max') + + prediction = layers.fc(input=[fc_last, lstm_last], + size=class_dim, + act='softmax') + cost = layers.cross_entropy(input=prediction, label=label) + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def main(): + BATCH_SIZE = 100 + PASS_NUM = 5 + + word_dict = paddle.dataset.imdb.word_dict() + print "load word dict successfully" + dict_dim = len(word_dict) + class_dim = 2 + + cost, acc = stacked_lstm_net(input_dim=dict_dim, class_dim=class_dim) + + train_data = paddle.batch( + paddle.reader.shuffle( + paddle.dataset.imdb.train(word_dict), buf_size=1000), + batch_size=BATCH_SIZE) + place = core.CPUPlace() + exe = Executor(place) + + exe.run(g_startup_program) + + for pass_id in xrange(PASS_NUM): + for data in train_data(): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([BATCH_SIZE, 1]) + + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if cost_val < 1.0 and acc_val > 0.7: + exit(0) + exit(1) + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py b/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py new file mode 100644 index 0000000000000000000000000000000000000000..26cbd01bc04916e53554e6f70bee7bcf25d6371c --- /dev/null +++ b/python/paddle/v2/framework/tests/book/test_understand_sentiment_lstm.py @@ -0,0 +1,107 @@ +import paddle.v2 as paddle +import paddle.v2.framework.layers as layers +import paddle.v2.framework.core as core +import paddle.v2.framework.optimizer as optimizer + +from paddle.v2.framework.framework import g_main_program, g_startup_program +from paddle.v2.framework.executor import Executor + +import numpy as np + + +def lstm_net(dict_dim, class_dim=2, emb_dim=32, seq_len=80, batch_size=50): + data = layers.data( + name="words", + shape=[seq_len * batch_size, 1], + append_batch_size=False, + data_type="int64") + label = layers.data( + name="label", + shape=[batch_size, 1], + append_batch_size=False, + data_type="int64") + + emb = layers.embedding(input=data, size=[dict_dim, emb_dim]) + emb = layers.reshape(x=emb, shape=[batch_size, seq_len, emb_dim]) + emb = layers.transpose(x=emb, axis=[1, 0, 2]) + + c_pre_init = layers.fill_constant( + dtype=emb.data_type, shape=[batch_size, emb_dim], value=0.0) + layer_1_out = layers.lstm(emb, c_pre_init=c_pre_init, hidden_dim=emb_dim) + layer_1_out = layers.transpose(x=layer_1_out, axis=[1, 0, 2]) + + prediction = layers.fc(input=layer_1_out, size=class_dim, act="softmax") + cost = layers.cross_entropy(input=prediction, label=label) + + avg_cost = layers.mean(x=cost) + adam_optimizer = optimizer.AdamOptimizer(learning_rate=0.002) + opts = adam_optimizer.minimize(avg_cost) + acc = layers.accuracy(input=prediction, label=label) + + return avg_cost, acc + + +def to_lodtensor(data, place): + seq_lens = [len(seq) for seq in data] + cur_len = 0 + lod = [cur_len] + for l in seq_lens: + cur_len += l + lod.append(cur_len) + flattened_data = np.concatenate(data, axis=0).astype("int64") + flattened_data = flattened_data.reshape([len(flattened_data), 1]) + res = core.LoDTensor() + res.set(flattened_data, place) + res.set_lod([lod]) + return res + + +def chop_data(data, chop_len=80, batch_len=50): + data = [(x[0][:chop_len], x[1]) for x in data if len(x[0]) >= chop_len] + + return data[:batch_len] + + +def prepare_feed_data(data, place): + tensor_words = to_lodtensor(map(lambda x: x[0], data), place) + + label = np.array(map(lambda x: x[1], data)).astype("int64") + label = label.reshape([50, 1]) + tensor_label = core.LoDTensor() + tensor_label.set(label, place) + + return tensor_words, tensor_label + + +def main(): + word_dict = paddle.dataset.imdb.word_dict() + cost, acc = lstm_net(dict_dim=len(word_dict), class_dim=2) + + batch_size = 100 + train_data = paddle.batch( + paddle.reader.buffered( + paddle.dataset.imdb.train(word_dict), size=batch_size * 10), + batch_size=batch_size) + + data = chop_data(next(train_data())) + + place = core.CPUPlace() + tensor_words, tensor_label = prepare_feed_data(data, place) + exe = Executor(place) + exe.run(g_startup_program) + + while True: + outs = exe.run(g_main_program, + feed={"words": tensor_words, + "label": tensor_label}, + fetch_list=[cost, acc]) + cost_val = np.array(outs[0]) + acc_val = np.array(outs[1]) + + print("cost=" + str(cost_val) + " acc=" + str(acc_val)) + if acc_val > 0.9: + break + + +if __name__ == '__main__': + main() diff --git a/python/paddle/v2/framework/tests/test_word2vec.py b/python/paddle/v2/framework/tests/book/test_word2vec.py similarity index 71% rename from python/paddle/v2/framework/tests/test_word2vec.py rename to python/paddle/v2/framework/tests/book/test_word2vec.py index 515d30d3e23edf429304d796faa8e17532168e26..cb9fc2ab62b56348db7a320f7d40d2f0a7bf9d21 100644 --- a/python/paddle/v2/framework/tests/test_word2vec.py +++ b/python/paddle/v2/framework/tests/book/test_word2vec.py @@ -3,13 +3,13 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.executor import Executor import numpy as np -init_program = Program() -program = Program() +startup_program = Program() +main_program = Program() embed_size = 32 hidden_size = 256 @@ -24,32 +24,32 @@ first_word = layers.data( name='firstw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) second_word = layers.data( name='secondw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) third_word = layers.data( name='thirdw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) forth_word = layers.data( name='forthw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) next_word = layers.data( name='nextw', shape=[1], data_type='int64', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_first = layers.embedding( input=first_word, @@ -57,16 +57,16 @@ embed_first = layers.embedding( data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_second = layers.embedding( input=second_word, size=[dict_size, embed_size], data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_third = layers.embedding( input=third_word, @@ -74,42 +74,43 @@ embed_third = layers.embedding( data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) embed_forth = layers.embedding( input=forth_word, size=[dict_size, embed_size], data_type='float32', is_sparse=is_sparse, param_attr={'name': 'shared_w'}, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) hidden1 = layers.fc(input=concat_embed, size=hidden_size, act='sigmoid', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) predict_word = layers.fc(input=hidden1, size=dict_size, act='softmax', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) cost = layers.cross_entropy( input=predict_word, label=next_word, - program=program, - init_program=init_program) -avg_cost = layers.mean(x=cost, program=program, init_program=init_program) + main_program=main_program, + startup_program=startup_program) +avg_cost = layers.mean( + x=cost, main_program=main_program, startup_program=startup_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) -opts = sgd_optimizer.minimize(avg_cost) +opts = sgd_optimizer.minimize(avg_cost, startup_program) train_reader = paddle.batch( paddle.dataset.imikolov.train(word_dict, N), batch_size) @@ -117,7 +118,11 @@ train_reader = paddle.batch( place = core.CPUPlace() exe = Executor(place) -exe.run(init_program, feed={}, fetch_list=[]) +# fix https://github.com/PaddlePaddle/Paddle/issues/5434 then remove +# below exit line. +exit(0) + +exe.run(startup_program, feed={}, fetch_list=[]) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): @@ -145,7 +150,7 @@ for pass_id in range(PASS_NUM): next_tensor = core.LoDTensor() next_tensor.set(next_data, place) - outs = exe.run(program, + outs = exe.run(main_program, feed={ 'firstw': first_tensor, 'secondw': second_tensor, diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 2e6710b5fcfe5a531067498e38a4cb93d3165602..4a269341a4be6c1b72fde5166b7dd089236700b8 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -215,7 +215,11 @@ class OpTest(unittest.TestCase): if isinstance(input_vars[var_name], list): for name, np_value in self.inputs[var_name]: tensor = core.LoDTensor() - tensor.set(np_value, place) + if isinstance(np_value, tuple): + tensor.set(np_value[0], place) + tensor.set_lod(np_value[1]) + else: + tensor.set(np_value, place) feed_map[name] = tensor else: tensor = core.LoDTensor() @@ -236,7 +240,6 @@ class OpTest(unittest.TestCase): inputs = append_input_output(block, op_proto, self.inputs, True) outputs = append_input_output(block, op_proto, self.outputs, False) - op = block.append_op( type=self.op_type, inputs=inputs, @@ -397,9 +400,11 @@ class OpTest(unittest.TestCase): if not isinstance(item[0], basestring): item = [[param_name] + list(item)] if len(item) == 2: - # only set var name and value, set lod to None - var[i] = list(item) + [None] - + if isinstance(item[1], tuple): + var[i] = [item[0], item[1][0], item[1][1]] + else: + # only set var name and value, set lod to None + var[i] = list(item) + [None] var_descs = [(block.create_var( name=name, shape=each.shape, dtype=each.dtype), each, lod) for name, each, lod in var] diff --git a/python/paddle/v2/framework/tests/test_array_read_write_op.py b/python/paddle/v2/framework/tests/test_array_read_write_op.py new file mode 100644 index 0000000000000000000000000000000000000000..79e9938216e2abda5432e525804b0bcb9a655655 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_array_read_write_op.py @@ -0,0 +1,91 @@ +import unittest +import paddle.v2.framework.core as core +import paddle.v2.framework.layers as layers +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +from paddle.v2.framework.framework import g_main_program +import numpy + + +class TestArrayReadWrite(unittest.TestCase): + def test_read_write(self): + x = [ + layers.data( + name='x0', shape=[100]), layers.data( + name='x1', shape=[100]), layers.data( + name='x2', shape=[100]) + ] + + for each_x in x: + each_x.stop_gradient = False + + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = False + arr = layers.array_write(x=x[0], i=i) + i = layers.increment(x=i) + arr = layers.array_write(x=x[1], i=i, array=arr) + i = layers.increment(x=i) + arr = layers.array_write(x=x[2], i=i, array=arr) + + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = False + a0 = layers.array_read(array=arr, i=i) + i = layers.increment(x=i) + a1 = layers.array_read(array=arr, i=i) + i = layers.increment(x=i) + a2 = layers.array_read(array=arr, i=i) + + mean_a0 = layers.mean(x=a0) + mean_a1 = layers.mean(x=a1) + mean_a2 = layers.mean(x=a2) + + a_sum = layers.sums(input=[mean_a0, mean_a1, mean_a2]) + + mean_x0 = layers.mean(x=x[0]) + mean_x1 = layers.mean(x=x[1]) + mean_x2 = layers.mean(x=x[2]) + + x_sum = layers.sums(input=[mean_x0, mean_x1, mean_x2]) + + scope = core.Scope() + cpu = core.CPUPlace() + + exe = Executor(cpu) + + tensor = core.LoDTensor() + tensor.set(numpy.random.random(size=(100, 100)).astype('float32'), cpu) + + outs = map(numpy.array, + exe.run(feed={'x0': tensor, + 'x1': tensor, + 'x2': tensor}, + fetch_list=[a_sum, x_sum], + scope=scope)) + self.assertEqual(outs[0], outs[1]) + + total_sum = layers.sums(input=[a_sum, x_sum]) + total_sum_scaled = layers.scale(x=total_sum, scale=1 / 6.0) + + append_backward_ops(total_sum_scaled) + + g_vars = map(g_main_program.global_block().var, + [each_x.name + "@GRAD" for each_x in x]) + g_out = [ + item.sum() + for item in map( + numpy.array, + exe.run(feed={'x0': tensor, + 'x1': tensor, + 'x2': tensor}, + fetch_list=g_vars)) + ] + g_out_sum = numpy.array(g_out).sum() + + # since our final gradient is 1 and the neural network are all linear + # with mean_op. + # the input gradient should also be 1 + self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_assign_op.py b/python/paddle/v2/framework/tests/test_assign_op.py new file mode 100644 index 0000000000000000000000000000000000000000..1b0c145f1a69678b228bc70e4e4e273f5bcf9888 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_assign_op.py @@ -0,0 +1,21 @@ +import op_test +import numpy +import unittest + + +class TestAssignOp(op_test.OpTest): + def setUp(self): + self.op_type = "assign" + x = numpy.random.random(size=(100, 10)) + self.inputs = {'X': x} + self.outputs = {'Out': x} + + def test_forward(self): + self.check_output() + + def test_backward(self): + self.check_grad(['X'], 'Out') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py b/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py new file mode 100644 index 0000000000000000000000000000000000000000..080ca43b8269e0f6a9f4d0ce3973f4d4a07a8e2a --- /dev/null +++ b/python/paddle/v2/framework/tests/test_bilinear_tensor_product_op.py @@ -0,0 +1,37 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestBilinearTensorProductOp(OpTest): + def setUp(self): + self.op_type = "bilinear_tensor_product" + batch_size = 6 + size0 = 3 + size1 = 4 + size2 = 5 + a = np.random.random((batch_size, size0)).astype("float32") + b = np.random.random((batch_size, size1)).astype("float32") + w = np.random.random((size2, size0, size1)).astype("float32") + bias = np.random.random((1, size2)).astype("float32") + output = np.zeros((batch_size, size2)).astype("float32") + for i in range(size2): + w_i = w[i, :, :] + output[:, i] = np.sum(np.matmul(a, w_i) * b, axis=1) + self.inputs = { + 'X': a, + 'Y': b, + 'Weight': w, + 'Bias': bias, + } + self.outputs = {'Out': output + bias} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y', 'Weight', 'Bias'], 'Out') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_chunk_eval_op.py b/python/paddle/v2/framework/tests/test_chunk_eval_op.py new file mode 100644 index 0000000000000000000000000000000000000000..48673296a67716c4de804da533f0fd2567f10e2e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_chunk_eval_op.py @@ -0,0 +1,179 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class Segment(object): + def __init__(self, chunk_type, start_idx, end_idx): + self.chunk_type = chunk_type + self.start_idx = start_idx + self.end_idx = end_idx + + def __str__(self): + return '(Segment: %s, %s, %s)' % (self.chunk_type, self.start_idx, + self.end_idx) + + __repr__ = __str__ + + +class TestChunkEvalOp(OpTest): + num_sequences = 5 + batch_size = 50 + + def parse_scheme(self): + if self.scheme == 'IOB': + self.num_tag_types = 2 + elif self.scheme == 'IOE': + self.num_tag_types = 2 + + def fill_with_chunks(self, data, chunks): + for chunk in chunks: + if self.scheme == 'IOB': + data[chunk.start_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.start_idx + 1: + chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1) + data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1 + ) if chunk.start_idx < chunk.end_idx else data[chunk.start_idx] + elif self.scheme == 'IOE': + data[chunk.start_idx: + chunk.end_idx] = chunk.chunk_type * self.num_tag_types + data[chunk.end_idx] = chunk.chunk_type * self.num_tag_types + ( + self.num_tag_types - 1) + + def rand_chunks(self, starts, num_chunks): + if num_chunks < 0: + num_chunks = np.random.randint(starts[-1]) + chunks = [] + # generate chunk beginnings + chunk_begins = sorted( + np.random.choice( + range(starts[-1]), num_chunks, replace=False)) + seq_chunk_begins = [] + begin_idx = 0 + # divide chunks into sequences + for i in range(len(starts) - 1): + tmp_chunk_begins = [] + while begin_idx < len(chunk_begins) and chunk_begins[ + begin_idx] < starts[i + 1]: + tmp_chunk_begins.append(chunk_begins[begin_idx]) + begin_idx += 1 + seq_chunk_begins.append(tmp_chunk_begins) + # generate chunk ends + chunk_ends = [] + for i in range(len(seq_chunk_begins)): + for j in range(len(seq_chunk_begins[i])): + low = seq_chunk_begins[i][j] + high = seq_chunk_begins[i][j + 1] if j < len(seq_chunk_begins[ + i]) - 1 else starts[i + 1] + chunk_ends.append(np.random.randint(low, high)) + # generate chunks + for chunk_pos in zip(chunk_begins, chunk_ends): + chunk_type = np.random.randint(self.num_chunk_types) + chunks.append(Segment(chunk_type, *chunk_pos)) + return chunks + + def gen_chunks(self, infer, label, starts): + chunks = self.rand_chunks(starts, + self.num_infer_chunks + self.num_label_chunks + - self.num_correct_chunks) + correct_chunks = np.random.choice( + range(len(chunks)), self.num_correct_chunks, replace=False) + infer_chunks = np.random.choice( + [x for x in range(len(chunks)) if x not in correct_chunks], + self.num_infer_chunks - self.num_correct_chunks, + replace=False) + infer_chunks = sorted(correct_chunks.tolist() + infer_chunks.tolist()) + label_chunks = np.random.choice( + [x for x in range(len(chunks)) if x not in infer_chunks], + self.num_label_chunks - self.num_correct_chunks, + replace=False) + label_chunks = sorted(correct_chunks.tolist() + label_chunks.tolist()) + self.fill_with_chunks(infer, [chunks[idx] for idx in infer_chunks]) + self.fill_with_chunks(label, [chunks[idx] for idx in label_chunks]) + # exclude types in excluded_chunk_types + if len(self.excluded_chunk_types) > 0: + for idx in correct_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_correct_chunks -= 1 + for idx in infer_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_infer_chunks -= 1 + for idx in label_chunks: + if chunks[idx].chunk_type in self.excluded_chunk_types: + self.num_label_chunks -= 1 + return self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks + + def set_confs(self): + # Use the IOB scheme and labels with 2 chunk types + self.scheme = 'IOB' + self.num_chunk_types = 2 + self.excluded_chunk_types = [] + self.other_chunk_type = self.num_chunk_types + self.attrs = { + 'num_chunk_types': self.num_chunk_types, + 'chunk_scheme': self.scheme, + 'excluded_chunk_types': self.excluded_chunk_types + } + self.parse_scheme() + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 4, 5, 9 + + def set_data(self): + infer = np.zeros((self.batch_size, )).astype('int32') + infer.fill(self.num_chunk_types * self.num_tag_types) + label = np.copy(infer) + starts = np.random.choice( + range(1, self.batch_size), self.num_sequences - 1, + replace=False).tolist() + starts.extend([0, self.batch_size]) + starts = sorted(starts) + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = self.gen_chunks( + infer, label, starts) + self.inputs = { + 'Inference': (infer, [starts]), + 'Label': (label, [starts]) + } + precision = float( + self.num_correct_chunks + ) / self.num_infer_chunks if self.num_infer_chunks else 0 + recall = float(self.num_correct_chunks + ) / self.num_label_chunks if self.num_label_chunks else 0 + f1 = float(2 * precision * recall) / ( + precision + recall) if self.num_correct_chunks else 0 + self.outputs = { + 'Precision': np.asarray( + [precision], dtype='float32'), + 'Recall': np.asarray( + [recall], dtype='float32'), + 'F1-Score': np.asarray( + [f1], dtype='float32') + } + + def setUp(self): + self.op_type = 'chunk_eval' + self.set_confs() + self.set_data() + + def test_check_output(self): + self.check_output() + + +class TestChunkEvalOpWithExclude(TestChunkEvalOp): + def set_confs(self): + # Use the IOE scheme and labels with 3 chunk types + self.scheme = 'IOE' + self.num_chunk_types = 3 + self.excluded_chunk_types = [1] + self.other_chunk_type = self.num_chunk_types + self.attrs = { + 'num_chunk_types': self.num_chunk_types, + 'chunk_scheme': self.scheme, + 'excluded_chunk_types': self.excluded_chunk_types + } + self.parse_scheme() + self.num_correct_chunks, self.num_infer_chunks, self.num_label_chunks = 15, 18, 20 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_clip_by_norm_op.py b/python/paddle/v2/framework/tests/test_clip_by_norm_op.py new file mode 100644 index 0000000000000000000000000000000000000000..02f6108a3a661b0e32cd2e7ed65cb4b8cb50c067 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_clip_by_norm_op.py @@ -0,0 +1,50 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestClipByNormOp(OpTest): + def setUp(self): + self.max_relative_error = 0.006 + self.initTestCase() + input = np.random.random(self.shape).astype("float32") + input[np.abs(input) < self.max_relative_error] = 0.5 + self.op_type = "clip_by_norm" + self.inputs = {'X': input, } + self.attrs = {} + self.attrs['max_norm'] = self.max_norm + norm = np.sqrt(np.sum(np.square(input))) + if norm > self.max_norm: + output = self.max_norm * input / norm + else: + output = input + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def initTestCase(self): + self.shape = (100, ) + self.max_norm = 1.0 + + +class TestCase1(TestClipByNormOp): + def initTestCase(self): + self.shape = (100, ) + self.max_norm = 1e20 + + +class TestCase2(TestClipByNormOp): + def initTestCase(self): + self.shape = (16, 16) + self.max_norm = 0.1 + + +class TestCase3(TestClipByNormOp): + def initTestCase(self): + self.shape = (4, 8, 16) + self.max_norm = 1.0 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_compare_op.py b/python/paddle/v2/framework/tests/test_compare_op.py new file mode 100644 index 0000000000000000000000000000000000000000..bb0256694d77323f12c50856533e93b090dc6198 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_compare_op.py @@ -0,0 +1,29 @@ +import op_test +import unittest +import numpy + + +def create_test_class(op_type, typename, callback): + class Cls(op_test.OpTest): + def setUp(self): + a = numpy.random.random(size=(10, 7)).astype(typename) + b = numpy.random.random(size=(10, 7)).astype(typename) + c = callback(a, b) + self.inputs = {'X': a, 'Y': b} + self.outputs = {'Out': c} + self.op_type = op_type + + def test_output(self): + self.check_output() + + cls_name = "{0}_{1}".format(op_type, typename) + Cls.__name__ = cls_name + globals()[cls_name] = Cls + + +for _type_name in {'float32', 'float64', 'int32', 'int64'}: + create_test_class('less_than', _type_name, lambda _a, _b: _a < _b) + create_test_class('equal', _type_name, lambda _a, _b: _a == _b) + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conditional_block.py b/python/paddle/v2/framework/tests/test_conditional_block.py new file mode 100644 index 0000000000000000000000000000000000000000..9b96ff306c37a6f96bd78951752d32274cca6282 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conditional_block.py @@ -0,0 +1,40 @@ +import unittest +import paddle.v2.framework.layers as layers +import paddle.v2.framework.core as core +from paddle.v2.framework.framework import g_startup_program, g_main_program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +import numpy + + +class ConditionalBlock(unittest.TestCase): + def test_forward(self): + data = layers.data(name='X', shape=[1], data_type='float32') + data.stop_gradient = False + cond = layers.ConditionalBlock(inputs=[data]) + out = layers.create_tensor(dtype='float32') + with cond.block(): + hidden = layers.fc(input=data, size=10) + layers.assign(hidden, out) + + cpu = core.CPUPlace() + exe = Executor(cpu) + exe.run(g_startup_program) + + x = core.LoDTensor() + x.set(numpy.random.random(size=(10, 1)).astype('float32'), cpu) + + outs = map(numpy.array, exe.run(feed={'X': x}, fetch_list=[out]))[0] + print outs + loss = layers.mean(x=out) + append_backward_ops(loss=loss) + outs = map(numpy.array, + exe.run(feed={'X': x}, + fetch_list=[ + g_main_program.block(0).var(data.name + "@GRAD") + ]))[0] + print outs + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv2d_op.py b/python/paddle/v2/framework/tests/test_conv2d_op.py index f58b96463cf78103b2acb3d80652ef0aa988ad49..04ae7f294c27fdceaaff2e9a7ed854213e643945 100644 --- a/python/paddle/v2/framework/tests/test_conv2d_op.py +++ b/python/paddle/v2/framework/tests/test_conv2d_op.py @@ -61,25 +61,23 @@ class TestConv2dOp(OpTest): def test_check_grad(self): self.check_grad( - set(['Input', 'Filter']), 'Output', max_relative_error=0.05) + set(['Input', 'Filter']), 'Output', max_relative_error=0.02) def test_check_grad_no_filter(self): self.check_grad( ['Input'], 'Output', - max_relative_error=0.05, + max_relative_error=0.02, no_grad_set=set(['Filter'])) def test_check_grad_no_input(self): self.check_grad( ['Filter'], 'Output', - max_relative_error=0.05, + max_relative_error=0.02, no_grad_set=set(['Input'])) def init_test_case(self): - # self.groups = 1 - # self.op_type = "conv2d" self.pad = [0, 0] self.stride = [1, 1] self.dilations = [1, 1] @@ -103,6 +101,9 @@ class TestWithGroup(TestConv2dOp): self.op_type = "conv2d" +#----------------Conv2dCudnn---------------- + + class TestCudnn(TestConv2dOp): def init_group(self): self.groups = 1 diff --git a/python/paddle/v2/framework/tests/test_conv2dtranspose_op.py b/python/paddle/v2/framework/tests/test_conv2d_transpose_op.py similarity index 87% rename from python/paddle/v2/framework/tests/test_conv2dtranspose_op.py rename to python/paddle/v2/framework/tests/test_conv2d_transpose_op.py index 53604c58b70a534dff6b0a668d380fb8f10f53f6..54349c018c4a53b8767d6cd4f94d99c719dc0237 100644 --- a/python/paddle/v2/framework/tests/test_conv2dtranspose_op.py +++ b/python/paddle/v2/framework/tests/test_conv2d_transpose_op.py @@ -45,37 +45,36 @@ class TestConv2dTransposeOp(OpTest): filter_ = np.random.random(self.filter_size).astype("float32") output = conv2dtranspose_forward_naive( input_, filter_, conv2dtranspose_param).astype('float32') - # print 'deconv output py', output, output.shape self.inputs = {'Input': input_, 'Filter': filter_} self.attrs = { 'strides': self.stride, 'paddings': self.pad, - # 'dilations': self.dilations + 'dilations': self.dilations } self.outputs = {'Output': output} def test_check_output(self): - print 'check output here' + print 'check output here for', self.op_type self.check_output() - def test_check_grad(self): + def test_check_grad_no_input(self): self.check_grad( - set(['Input', 'Filter']), 'Output', max_relative_error=0.05) + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input'])) def test_check_grad_no_filter(self): self.check_grad( ['Input'], 'Output', - max_relative_error=0.05, + max_relative_error=0.02, no_grad_set=set(['Filter'])) - def test_check_grad_no_input(self): + def test_check_grad(self): self.check_grad( - ['Filter'], - 'Output', - max_relative_error=0.05, - no_grad_set=set(['Input'])) + set(['Input', 'Filter']), 'Output', max_relative_error=0.02) def init_test_case(self): self.pad = [0, 0] @@ -86,17 +85,14 @@ class TestConv2dTransposeOp(OpTest): self.filter_size = [f_c, 6, 3, 3] def init_op_type(self): - self.op_type = "conv2dtranspose" - + self.op_type = "conv2d_transpose" -""" -class TestCudnn(TestConv2dOp): - def init_group(self): - self.groups = 1 +# ------------ test_cudnn ------------ +class TestCudnn(TestConv2dTransposeOp): def init_op_type(self): - self.op_type = "conv_cudnn" -""" + self.op_type = "conv2d_transpose_cudnn" + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv3d_op.py b/python/paddle/v2/framework/tests/test_conv3d_op.py new file mode 100644 index 0000000000000000000000000000000000000000..44c192f58d25f8ddaa38d2ba7c7c19b9a5bd7dc1 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conv3d_op.py @@ -0,0 +1,131 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def conv3d_forward_naive(input, filter, group, conv_param): + in_n, in_c, in_d, in_h, in_w = input.shape + out_c, f_c, f_d, f_h, f_w = filter.shape + assert f_c * group == in_c + assert np.mod(out_c, group) == 0 + sub_out_c = out_c / group + + stride, pad = conv_param['stride'], conv_param['pad'] + out_d = 1 + (in_d + 2 * pad[0] - f_h) / stride[0] + out_h = 1 + (in_h + 2 * pad[1] - f_h) / stride[1] + out_w = 1 + (in_w + 2 * pad[2] - f_w) / stride[2] + out = np.zeros((in_n, out_c, out_d, out_h, out_w)) + + input_pad = np.pad(input, ((0, ), (0, ), (pad[0], ), (pad[1], ), + (pad[2], )), + mode='constant', + constant_values=0) + for d in range(out_d): + for i in range(out_h): + for j in range(out_w): + for g in range(group): + input_pad_masked = \ + input_pad[:, g * f_c:(g + 1) * f_c, + d * stride[0]:d * stride[0] + f_d, + i * stride[1]:i * stride[1] + f_h, + j * stride[2]:j * stride[2] + f_w] + f_sub = filter[g * sub_out_c:(g + 1) * + sub_out_c, :, :, :, :] + for k in range(sub_out_c): + out[:, g * sub_out_c + k, d, i, j] = \ + np.sum(input_pad_masked * f_sub[k, :, :, :, :], + axis=(1, 2, 3, 4)) + + return out + + +class TestConv3dOp(OpTest): + def setUp(self): + self.init_group() + self.init_op_type() + self.init_test_case() + + conv3d_param = {'stride': self.stride, 'pad': self.pad} + input = np.random.random(self.input_size).astype("float32") + filter = np.random.random(self.filter_size).astype("float32") + output = conv3d_forward_naive(input, filter, self.groups, + conv3d_param).astype("float32") + + self.inputs = {'Input': input, 'Filter': filter} + self.attrs = { + 'strides': self.stride, + 'paddings': self.pad, + 'groups': self.groups + } + self.outputs = {'Output': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.03) + + def test_check_grad_no_filter(self): + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Filter'])) + + def test_check_grad_no_input(self): + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.03, + no_grad_set=set(['Input'])) + + def init_test_case(self): + self.pad = [0, 0, 0] + self.stride = [1, 1, 1] + self.input_size = [2, 3, 4, 4, 4] # NCDHW + assert np.mod(self.input_size[1], self.groups) == 0 + f_c = self.input_size[1] / self.groups + self.filter_size = [6, f_c, 3, 3, 3] + + def init_group(self): + self.groups = 1 + + def init_op_type(self): + self.op_type = "conv3d" + + +class TestCase1(TestConv3dOp): + def init_test_case(self): + self.pad = [1, 1, 1] + self.stride = [1, 1, 1] + self.input_size = [2, 3, 4, 4, 4] # NCDHW + assert np.mod(self.input_size[1], self.groups) == 0 + f_c = self.input_size[1] / self.groups + self.filter_size = [6, f_c, 3, 3, 3] + + def init_group(self): + self.groups = 1 + + def init_op_type(self): + self.op_type = "conv3d" + + +class TestWithGroup1(TestConv3dOp): + def init_group(self): + self.groups = 3 + + def init_op_type(self): + self.op_type = "conv3d" + + +class TestWithGroup2(TestCase1): + def init_group(self): + self.groups = 3 + + def init_op_type(self): + self.op_type = "conv3d" + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv3d_transpose_op.py b/python/paddle/v2/framework/tests/test_conv3d_transpose_op.py new file mode 100644 index 0000000000000000000000000000000000000000..132fe7931438a30cf02e4ad2894c0838e48ffc9f --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conv3d_transpose_op.py @@ -0,0 +1,97 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def conv3dtranspose_forward_naive(input_, filter_, conv3dtranspose_param): + # [2, 3, 5, 5, 5] + in_n, in_c, in_d, in_h, in_w = input_.shape + # [3, 6, 3, 3, 3] + f_c, out_c, f_d, f_h, f_w = filter_.shape + assert in_c == f_c + + stride, pad = conv3dtranspose_param['stride'], conv3dtranspose_param['pad'] + out_d = (in_d - 1) * stride[0] + f_d + out_h = (in_h - 1) * stride[1] + f_h + out_w = (in_w - 1) * stride[2] + f_w + + out = np.zeros((in_n, out_c, out_d, out_h, out_w)) + + for n in range(in_n): + for d in range(in_d): + for i in range(in_h): + for j in range(in_w): + input_masked = input_[n, :, d, i, j] # (c) + input_masked = np.reshape(input_masked, (in_c, 1, 1, 1)) + input_masked = np.tile(input_masked, (1, f_d, f_h, f_w)) + + for k in range(out_c): + tmp_out = np.sum(input_masked * filter_[:, k, :, :, :], + axis=0) + d1, d2 = d * stride[0], d * stride[0] + f_d + i1, i2 = i * stride[1], i * stride[1] + f_h + j1, j2 = j * stride[2], j * stride[2] + f_w + out[n, k, d1:d2, i1:i2, j1:j2] += tmp_out + + return out + + +class TestConv3dTransposeOp(OpTest): + def setUp(self): + # init as conv transpose + self.init_op_type() + + # [2, 3, 5, 5, 5] -> kernel [3, 6, 3, 3, 3] -> output [2, 6, 7, 7, 7] + self.init_test_case() + + conv3dtranspose_param = {'stride': self.stride, 'pad': self.pad} + input_ = np.random.random(self.input_size).astype("float32") + filter_ = np.random.random(self.filter_size).astype("float32") + output = conv3dtranspose_forward_naive( + input_, filter_, conv3dtranspose_param).astype("float32") + # print 'deconv output py', output, output.shape + + self.inputs = {'Input': input_, 'Filter': filter_} + self.attrs = { + 'strides': self.stride, + 'paddings': self.pad, + # 'dilations': self.dilations + } + self.outputs = {'Output': output} + + def test_check_output(self): + print 'check output here' + self.check_output() + + def test_check_grad(self): + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.02) + + def test_check_grad_no_filter(self): + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter'])) + + def test_check_grad_no_input(self): + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input'])) + + def init_test_case(self): + self.pad = [0, 0, 0] + self.stride = [1, 1, 1] + self.dilations = [1, 1, 1] + self.input_size = [2, 3, 5, 5, 5] # NCHW + f_c = self.input_size[1] + self.filter_size = [f_c, 6, 3, 3, 3] + + def init_op_type(self): + self.op_type = "conv3d_transpose" + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_create_op_doc_string.py b/python/paddle/v2/framework/tests/test_create_op_doc_string.py new file mode 100644 index 0000000000000000000000000000000000000000..d21e96df2a64d3fa418dca94690ea0b820db80de --- /dev/null +++ b/python/paddle/v2/framework/tests/test_create_op_doc_string.py @@ -0,0 +1,11 @@ +import unittest +import paddle.v2.framework.layers as layers + + +class TestDocString(unittest.TestCase): + def test_layer_doc_string(self): + print layers.dropout.__doc__ + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_crf_decoding_op.py b/python/paddle/v2/framework/tests/test_crf_decoding_op.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2b996bf430d5a0edaa0de459a937adffd9f8f6 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_crf_decoding_op.py @@ -0,0 +1,146 @@ +import unittest +import random +import numpy as np + +from op_test import OpTest + + +class CRFDecoding(object): + def __init__(self, emission_weights, transition_weights, + seq_start_positions): + assert (emission_weights.shape[0] == seq_start_positions[-1]) + self.tag_num = emission_weights.shape[1] + self.seq_num = len(seq_start_positions) - 1 + + self.seq_start_positions = seq_start_positions + self.x = emission_weights + + self.a = transition_weights[0, :] + self.b = transition_weights[1, :] + self.w = transition_weights[2:, :] + + self.track = np.zeros( + (seq_start_positions[-1], self.tag_num), dtype="int32") + self.decoded_path = np.zeros( + (seq_start_positions[-1], 1), dtype="int32") + + def _decode_one_sequence(self, decoded_path, x): + seq_len, tag_num = x.shape + alpha = np.zeros((seq_len, tag_num), dtype="float64") + track = np.zeros((seq_len, tag_num), dtype="int32") + + for i in range(tag_num): + alpha[0, i] = self.a[i] + x[0, i] + + for k in range(1, seq_len): + for i in range(tag_num): + max_score = -np.finfo("float64").max + max_idx = 0 + for j in range(tag_num): + score = alpha[k - 1, j] + self.w[j, i] + if score > max_score: + max_score = score + max_idx = j + alpha[k, i] = max_score + x[k, i] + track[k, i] = max_idx + + max_score = -np.finfo("float64").max + max_idx = 0 + for i in range(tag_num): + score = alpha[seq_len - 1, i] + self.b[i] + if score > max_score: + max_score = score + max_idx = i + + decoded_path[-1] = max_idx + for i in range(seq_len - 1, 0, -1): + decoded_path[i - 1] = max_idx = track[i, max_idx] + + def decode(self): + for i in range(self.seq_num): + start = self.seq_start_positions[i] + end = self.seq_start_positions[i + 1] + self._decode_one_sequence(self.decoded_path[start:end, :], + self.x[start:end, :]) + return self.decoded_path + + +class TestCRFDecodingOp1(OpTest): + """ + Compare the dynamic program with random generated parameters and inputs + with grouth truth not being given. + """ + + def set_test_data(self): + SEQ_NUM = 3 + TAG_NUM = 17 + MAX_SEQ_LEN = 10 + + lod = [[0]] + for i in range(SEQ_NUM): + lod[-1].append(lod[-1][-1] + random.randint(1, MAX_SEQ_LEN)) + emission = np.random.uniform(-1, 1, + [lod[-1][-1], TAG_NUM]).astype("float64") + transition = np.random.uniform(-0.5, 0.5, + [TAG_NUM + 2, TAG_NUM]).astype("float64") + + self.inputs = { + "Emission": (emission, lod), + "Transition": transition, + } + + decoder = CRFDecoding(emission, transition, lod[0]) + decoded_path = decoder.decode() + + self.outputs = {"ViterbiPath": decoded_path} + + def setUp(self): + self.op_type = "crf_decoding" + self.set_test_data() + + def test_check_output(self): + self.check_output() + + +class TestCRFDecodingOp2(OpTest): + """ + Compare the dynamic program with brute force computation with + ground truth being given. + """ + + def setUp(self): + self.op_type = "crf_decoding" + TAG_NUM = 5 + + lod = [[0, 1, 3, 6, 10]] + transition = np.repeat( + np.arange( + TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + TAG_NUM + 2, + axis=0) + emission = np.repeat( + np.arange( + TAG_NUM, dtype="float64").reshape(1, TAG_NUM), + lod[-1][-1], + axis=0) + + labels = np.random.randint( + low=0, high=TAG_NUM, size=(lod[-1][-1], 1), dtype="int32") + predicted_labels = np.ones( + (lod[-1][-1], 1), dtype="int32") * (TAG_NUM - 1) + expected_output = (labels == predicted_labels).astype("int32") + + self.inputs = { + "Emission": (emission, lod), + "Transition": transition, + "Label": (labels, lod) + } + + self.outputs = {"ViterbiPath": expected_output} + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_evaluator.py b/python/paddle/v2/framework/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..37dbfbc06bcd0da7e11924a048679c74a1cfb373 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_evaluator.py @@ -0,0 +1,64 @@ +from paddle.v2.framework.evaluator import Evaluator +from paddle.v2.framework.op import Operator +import paddle.v2.framework.core as core +import unittest +import op_test +import numpy as np + + +class TestEvaluator(unittest.TestCase): + def setup(self, scope, inputs, outputs): + def __create_var__(var_name, arr): + np_arr = np.array(arr) + scope.var(var_name) + # tensor = var.get_tensor() + # tensor.set_dims(np_arr.shape) + + for var_name, arr in inputs.iteritems(): + __create_var__(var_name, arr) + + for var_name, arr in outputs.iteritems(): + __create_var__(var_name, arr) + + def test_evaluator(self): + + inputs = { + 'Inference': np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 1]]).T, + 'Label': np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0]) + } + outputs = {'Accuracy': np.array([0.9])} + out_name = 'Accuracy' + + places = [core.CPUPlace()] + if core.is_compile_gpu(): + places.append(core.GPUPlace(0)) + + for place in places: + scope = core.Scope() + self.setup(scope, inputs, outputs) + + evaluator = Evaluator( + scope, + operator='accuracy', + input='Inference', + label='Label', + output=out_name, + place=place) + op_test.set_input(scope, evaluator.op, inputs, place) + ctx = core.DeviceContext.create(place) + + for i in range(10): # simulate 10 mini-batches + evaluator.evaluate(ctx) + + actual = np.array(scope.find_var(out_name).get_tensor()) + print actual + + self.assertTrue( + np.allclose( + actual, outputs[out_name], atol=1e-5), + "output name: " + out_name + " has diff.") + + +if __name__ == '__main__': + exit(0) + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_executor_and_mul.py b/python/paddle/v2/framework/tests/test_executor_and_mul.py index 35f775711167ce0d210044ab4cb382db802f39a5..c885cfbebd4b665ddf50adbc43673942dc949a0b 100644 --- a/python/paddle/v2/framework/tests/test_executor_and_mul.py +++ b/python/paddle/v2/framework/tests/test_executor_and_mul.py @@ -2,7 +2,7 @@ import unittest from paddle.v2.framework.layers import mul, data import paddle.v2.framework.core as core from paddle.v2.framework.executor import Executor -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program import numpy @@ -23,7 +23,7 @@ class TestExecutor(unittest.TestCase): tensor_b = core.LoDTensor() tensor_b.set(b_np, place) exe = Executor(place) - outs = exe.run(g_program, + outs = exe.run(g_main_program, feed={'a': tensor_a, 'b': tensor_b}, fetch_list=[out]) diff --git a/python/paddle/v2/framework/tests/test_expand_op.py b/python/paddle/v2/framework/tests/test_expand_op.py new file mode 100644 index 0000000000000000000000000000000000000000..0440f7a2bb159bab4923683b5d0980e59e0a69c9 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_expand_op.py @@ -0,0 +1,97 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestExpandOpRank1(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random(12).astype("float32")} + self.attrs = {'expand_times': [2]} + output = np.tile(self.inputs['X'], 2) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank2_Corner(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((12, 14)).astype("float32")} + self.attrs = {'expand_times': [1, 1]} + output = np.tile(self.inputs['X'], (1, 1)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank2(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((12, 14)).astype("float32")} + self.attrs = {'expand_times': [2, 3]} + output = np.tile(self.inputs['X'], (2, 3)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank3_Corner(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5)).astype("float32")} + self.attrs = {'expand_times': [1, 1, 1]} + output = np.tile(self.inputs['X'], (1, 1, 1)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank3(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5)).astype("float32")} + self.attrs = {'expand_times': [2, 1, 4]} + output = np.tile(self.inputs['X'], (2, 1, 4)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestExpandOpRank4(OpTest): + def setUp(self): + self.op_type = "expand" + self.inputs = {'X': np.random.random((2, 4, 5, 7)).astype("float32")} + self.attrs = {'expand_times': [3, 2, 1, 2]} + output = np.tile(self.inputs['X'], (3, 2, 1, 2)) + self.outputs = {'Out': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py b/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py index 065a9133dca25fac988f9493c1527e0d8f9821dc..99de6b5d052b41499800afb6181a235da340bc15 100644 --- a/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/v2/framework/tests/test_fill_constant_batch_size_like_op.py @@ -3,13 +3,32 @@ import numpy as np from op_test import OpTest -class TestFillConstantBatchSizeLikeOp(OpTest): +class TestFillConstantBatchSizeLikeWhenFirstDimIsBatchSize(OpTest): def setUp(self): self.op_type = "fill_constant_batch_size_like" self.inputs = {'Input': np.random.random((219, 232)).astype("float32")} - self.attrs = {'value': 3.5, 'shape': [-1, 132, 777]} + self.attrs = {'value': 3.5, 'shape': [-1, 132, 7]} - out = np.random.random((219, 132, 777)).astype("float32") + out = np.random.random((219, 132, 7)).astype("float32") + out.fill(3.5) + self.outputs = {'Out': out} + + def test_check_output(self): + self.check_output() + + +class TestFillConstantBatchSizeLikeWhenSecondDimIsBatchSize(OpTest): + def setUp(self): + self.op_type = "fill_constant_batch_size_like" + self.inputs = {'Input': np.random.random((219, 232)).astype("float32")} + self.attrs = { + 'value': 3.5, + 'shape': [132, -1, 7], + 'input_dim_idx': 0, + 'output_dim_idx': 1 + } + + out = np.random.random((132, 219, 7)).astype("float32") out.fill(3.5) self.outputs = {'Out': out} diff --git a/python/paddle/v2/framework/tests/test_framework_debug_str.py b/python/paddle/v2/framework/tests/test_framework_debug_str.py new file mode 100644 index 0000000000000000000000000000000000000000..8fdf8f91171ee334fac93c05a4d49056fa0e803d --- /dev/null +++ b/python/paddle/v2/framework/tests/test_framework_debug_str.py @@ -0,0 +1,13 @@ +import unittest +from paddle.v2.framework.framework import Program + + +class TestDebugStringFramework(unittest.TestCase): + def test_debug_str(self): + p = Program() + p.current_block().create_var(name='t', shape=[0, 1]) + self.assertRaises(ValueError, callableObj=p.__str__) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_gru_op.py b/python/paddle/v2/framework/tests/test_gru_op.py new file mode 100644 index 0000000000000000000000000000000000000000..b2474cff94c6c71cc62bc8e69a5d83e38d51c511 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_gru_op.py @@ -0,0 +1,156 @@ +import unittest +import numpy as np +import math +from op_test import OpTest +from test_lstm_op import identity, sigmoid, tanh, relu + + +class TestGRUOp(OpTest): + batch_size = 9 + frame_size = 5 + activate = { + 'identity': identity, + 'sigmoid': sigmoid, + 'tanh': tanh, + 'relu': relu + } + + @staticmethod + def seq_to_batch(lod, is_reverse): + idx_in_seq_list = [] + seq_starts = lod[0] + seq_lens = [] + for i in range(len(seq_starts) - 1): + seq_lens.append(seq_starts[i + 1] - seq_starts[i]) + sorted_seqs = sorted( + range(len(seq_lens)), lambda x, y: seq_lens[y] - seq_lens[x]) + num_batch = seq_lens[sorted_seqs[0]] + for batch_idx in range(num_batch): + idx_in_seq = [] + for i in range(len(seq_lens)): + if seq_lens[sorted_seqs[i]] <= batch_idx: + break + idx = (seq_starts[sorted_seqs[i] + 1] - 1 - batch_idx + ) if is_reverse else ( + seq_starts[sorted_seqs[i]] + batch_idx) + idx_in_seq.append(idx) + idx_in_seq_list.append(idx_in_seq) + return idx_in_seq_list + + def gru_step(self, x, h_p, w, b): + batch_size = x.shape[0] + frame_size = w.shape[0] + g = x + np.tile(b, (batch_size, 1)) + w_u_r = w.flatten()[:frame_size * frame_size * 2].reshape( + (frame_size, frame_size * 2)) + u_r = self.activate[self.attrs['gate_activation']](np.dot( + h_p, w_u_r) + g[:, :frame_size * 2]) + u = u_r[:, :frame_size] + r = u_r[:, frame_size:frame_size * 2] + r_h_p = r * h_p + w_c = w.flatten()[frame_size * frame_size * 2:].reshape( + (frame_size, frame_size)) + c = self.activate[self.attrs['activation']](np.dot(r_h_p, w_c) + + g[:, frame_size * 2:]) + g = np.hstack((u_r, c)) + h = u * c + (1 - u) * h_p + return g, r_h_p, h + + def gru(self): + input, lod = self.inputs['Input'] + w = self.inputs['Weight'] + b = self.inputs['Bias'] if self.inputs.has_key('Bias') else np.zeros( + (1, self.frame_size * 3)) + batch_gate = self.outputs['BatchGate'] + batch_reset_hidden_prev = self.outputs['BatchResetHiddenPrev'] + batch_hidden = self.outputs['BatchHidden'] + hidden = self.outputs['Hidden'] + idx_in_seq_list = self.idx_in_seq_list + h_p = self.inputs['H0'] if self.inputs.has_key('H0') else np.zeros( + (len(idx_in_seq_list[0]), self.frame_size)) + num_batch = len(idx_in_seq_list) + end_idx = 0 + for batch_idx in range(num_batch): + x = input[idx_in_seq_list[batch_idx]] + g, r_h_p, h = self.gru_step(x, h_p, w, b) + if batch_idx < (num_batch - 1): + h_p = h[:len(idx_in_seq_list[batch_idx + 1])] + start_idx = end_idx + end_idx = start_idx + len(idx_in_seq_list[batch_idx]) + batch_gate[start_idx:end_idx] = g + batch_reset_hidden_prev[start_idx:end_idx] = r_h_p + batch_hidden[start_idx:end_idx] = h + hidden[idx_in_seq_list[batch_idx]] = h + return batch_gate, batch_reset_hidden_prev, hidden + + def set_data(self): + lod = [[0, 2, 6, self.batch_size]] + self.idx_in_seq_list = self.seq_to_batch(lod, self.is_reverse) + batch_size = self.batch_size + frame_size = self.frame_size + input = np.random.rand(batch_size, frame_size * 3).astype('float64') + h0 = np.random.rand(len(self.idx_in_seq_list[0]), + frame_size).astype('float64') + weight = np.random.rand(frame_size, frame_size * 3).astype('float64') + bias = np.random.rand(1, frame_size * 3).astype('float64') + + self.inputs = { + 'Input': (input, lod), + 'H0': h0, + 'Weight': weight, + 'Bias': bias + } + + self.outputs = { + 'BatchGate': np.zeros( + (batch_size, frame_size * 3), dtype='float64'), + 'BatchResetHiddenPrev': np.zeros( + (batch_size, frame_size), dtype='float64'), + 'BatchHidden': np.zeros( + (batch_size, frame_size), dtype='float64'), + 'Hidden': np.zeros( + (batch_size, frame_size), dtype='float64') + } + + def set_confs(self): + self.is_reverse = False + self.attrs = { + 'activation': 'tanh', + 'gate_activation': 'sigmoid', + 'is_reverse': self.is_reverse + } + + def setUp(self): + self.op_type = "gru" + self.set_confs() + self.set_data() + self.gru() + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['Input', 'H0', 'Weight', 'Bias'], ['Hidden']) + + +class TestGRUOpNoInitial(TestGRUOp): + def set_data(self): + super(TestGRUOpNoInitial, self).set_data() + self.inputs.pop('H0') + + def test_check_grad(self): + self.check_grad(['Input', 'Weight', 'Bias'], ['Hidden']) + + +class TestGRUOpReverse(TestGRUOp): + def set_confs(self): + self.is_reverse = True + self.attrs = { + 'activation': 'identity', + 'gate_activation': 'sigmoid', + 'is_reverse': self.is_reverse + } + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_huber_loss_op.py b/python/paddle/v2/framework/tests/test_huber_loss_op.py index 003e7d7ed7ccdfc48b0aa8db0a6765b5c93e7c14..a24fcbec6cc4801118ce4ef97eb4692cd2351c28 100644 --- a/python/paddle/v2/framework/tests/test_huber_loss_op.py +++ b/python/paddle/v2/framework/tests/test_huber_loss_op.py @@ -21,7 +21,8 @@ class TestHuberLossOp(OpTest): 'Y': np.random.uniform(0, 1., (samples_num, 1)).astype('float32'), } residual = self.inputs['Y'] - self.inputs['X'] - loss = np.vectorize(huber_loss_forward)(residual, delta) + loss = np.vectorize(huber_loss_forward)(residual, + delta).astype('float32') self.attrs = {'delta': delta} self.outputs = { 'Residual': residual, @@ -43,6 +44,5 @@ class TestHuberLossOp(OpTest): ['X'], 'Out', max_relative_error=0.008, no_grad_set=set('residual')) -# TODO(typhoonzero): should add this back till we fix it -#if __name__ == '__main__': -# unittest.main() +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_image_classification_layer.py b/python/paddle/v2/framework/tests/test_image_classification_layer.py index b4eda13552e60f009ec910e3d21e9d77107133a1..b1a267ec32b1c937b946bee82e41b846ebbf1288 100644 --- a/python/paddle/v2/framework/tests/test_image_classification_layer.py +++ b/python/paddle/v2/framework/tests/test_image_classification_layer.py @@ -9,8 +9,8 @@ def conv_block(input, num_filter, groups, dropouts, - program=None, - init_program=None): + main_program=None, + startup_program=None): return nets.img_conv_group( input=input, pool_size=2, @@ -21,77 +21,81 @@ def conv_block(input, conv_with_batchnorm=True, conv_batchnorm_drop_rate=dropouts, pool_type='max', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) class TestLayer(unittest.TestCase): def test_batch_norm_layer(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program) + main_program=main_program) layers.batch_norm( - input=images, program=program, init_program=init_program) + input=images, + main_program=main_program, + startup_program=startup_program) - # print str(program) + # print str(main_program) def test_dropout_layer(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program) + main_program=main_program) layers.dropout( x=images, dropout_prob=0.5, - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) - # print str(program) + # print str(main_program) def test_img_conv_group(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() images = layers.data( name='pixel', shape=[3, 48, 48], data_type='float32', - program=program, - init_program=init_program) - conv1 = conv_block(images, 64, 2, [0.3, 0], program, init_program) - conv2 = conv_block(conv1, 256, 3, [0.4, 0.4, 0], program, init_program) + main_program=main_program, + startup_program=startup_program) + conv1 = conv_block(images, 64, 2, [0.3, 0], main_program, + startup_program) + conv2 = conv_block(conv1, 256, 3, [0.4, 0.4, 0], main_program, + startup_program) - # print str(program) + # print str(main_program) def test_elementwise_add_with_act(self): - program = Program() - init_program = Program() + main_program = Program() + startup_program = Program() image1 = layers.data( name='pixel1', shape=[3, 48, 48], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) image2 = layers.data( name='pixel2', shape=[3, 48, 48], data_type='float32', - program=program, - init_program=init_program) + main_program=main_program, + startup_program=startup_program) out = layers.elementwise_add( x=image1, y=image2, act='relu', - program=program, - init_program=init_program) - # print(program) + main_program=main_program, + startup_program=startup_program) + # print(main_program) if __name__ == '__main__': diff --git a/python/paddle/v2/framework/tests/test_increment_op.py b/python/paddle/v2/framework/tests/test_increment_op.py deleted file mode 100644 index e174272b05b9413cc2bc1e099c4dd17899829e76..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_increment_op.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestIncrementOpPositiveStep(OpTest): - """Test increment op with positive step - """ - - def setUp(self): - self.op_type = "increment" - self.inputs = {'X': np.random.random((10, 10)).astype("float32")} - self.attrs = {'step': 14.8} - self.outputs = {'Out': self.inputs['X'] + self.attrs['step']} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(['X'], 'Out') - - -class TestIncrementOpNegativeStep(OpTest): - """Test increment op with negative step - """ - - def setUp(self): - self.op_type = "increment" - self.inputs = {'X': np.random.random((10, 10)).astype("float32")} - self.attrs = {'step': -3.8} - self.outputs = {'Out': self.inputs['X'] + self.attrs['step']} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(['X'], 'Out') - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_inference_model_io.py b/python/paddle/v2/framework/tests/test_inference_model_io.py index 4487ab989f3c5da92e086c1fd395c3d776dce9a9..48984f86a1864baade58aeb8e35c6065cc2a4bbb 100644 --- a/python/paddle/v2/framework/tests/test_inference_model_io.py +++ b/python/paddle/v2/framework/tests/test_inference_model_io.py @@ -3,7 +3,7 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.core as core import paddle.v2.framework.optimizer as optimizer -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program from paddle.v2.framework.io import save_inference_model, load_inference_model import paddle.v2.framework.executor as executor import unittest @@ -20,31 +20,31 @@ class TestBook(unittest.TestCase): name='x', shape=[2], data_type='float32', - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) y = layers.data( name='y', shape=[1], data_type='float32', - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) y_predict = layers.fc(input=x, size=1, act=None, - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) cost = layers.square_error_cost( input=y_predict, label=y, - program=program, - init_program=init_program) + main_program=program, + startup_program=init_program) avg_cost = layers.mean( - x=cost, program=program, init_program=init_program) + x=cost, main_program=program, startup_program=init_program) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001) - opts = sgd_optimizer.minimize(avg_cost) + opts = sgd_optimizer.minimize(avg_cost, init_program) place = core.CPUPlace() exe = executor.Executor(place) diff --git a/python/paddle/v2/framework/tests/test_initializer.py b/python/paddle/v2/framework/tests/test_initializer.py index f28fc8a86c7c8e683e00249a2f73dbbe6d7be27c..bd4d2e39d770aebb7468d516f463533185ea8680 100644 --- a/python/paddle/v2/framework/tests/test_initializer.py +++ b/python/paddle/v2/framework/tests/test_initializer.py @@ -1,3 +1,4 @@ +import numpy as np import unittest import paddle.v2.framework.framework as framework @@ -116,5 +117,111 @@ class TestNormalInitializer(unittest.TestCase): self.assertEqual(init_op.attr('seed'), 123) +class TestXavierInitializer(unittest.TestCase): + def test_uniform_xavier_initializer(self): + """Test Xavier initializer with uniform distribution on + for matrix multiply. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_uniform_xavier_initializer_conv(self): + """Test Xavier initializer with uniform distribution on + for convolutions. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + receptive_field_size = float(15 * 20) + limit = np.sqrt(6.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_xavier_initializer(self): + """Test Xavier initializer with normal distribution on + for matrix multiply. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + std = np.sqrt(2.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_xavier_initializer_conv(self): + """Test Xavier initializer with normal distribution on + for convolutions. + """ + program = framework.Program() + block = program.global_block() + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + receptive_field_size = float(15 * 20) + std = np.sqrt(2.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_xavier_initializer_supplied_arguments(self): + """Test the Xavier initializer with supplied arguments + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer( + fan_in=12, fan_out=23, seed=134)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (12 + 23)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 134) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_layers.py b/python/paddle/v2/framework/tests/test_layers.py index 5cbe790e3f019f5dcf6b201c4744e7502141ed99..b42af5ea45d54723e96279f9e16f82a1d52ad236 100644 --- a/python/paddle/v2/framework/tests/test_layers.py +++ b/python/paddle/v2/framework/tests/test_layers.py @@ -1,6 +1,6 @@ import paddle.v2.framework.layers as layers import paddle.v2.framework.nets as nets -from paddle.v2.framework.framework import Program, g_program +from paddle.v2.framework.framework import Program import paddle.v2.framework.core as core import unittest @@ -9,15 +9,15 @@ class TestBook(unittest.TestCase): def test_fit_a_line(self): program = Program() x = layers.data( - name='x', shape=[13], data_type='float32', program=program) - y_predict = layers.fc(input=x, size=1, act=None, program=program) + name='x', shape=[13], data_type='float32', main_program=program) + y_predict = layers.fc(input=x, size=1, act=None, main_program=program) y = layers.data( - name='y', shape=[1], data_type='float32', program=program) + name='y', shape=[1], data_type='float32', main_program=program) cost = layers.square_error_cost( - input=y_predict, label=y, program=program) + input=y_predict, label=y, main_program=program) - avg_cost = layers.mean(x=cost, program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) program.append_backward(avg_cost) print str(program) @@ -27,26 +27,42 @@ class TestBook(unittest.TestCase): # Change g_program, so the rest layers use `g_program` images = layers.data( - name='pixel', shape=[784], data_type='float32', program=program) + name='pixel', + shape=[784], + data_type='float32', + main_program=program) label = layers.data( - name='label', shape=[1], data_type='int32', program=program) - hidden1 = layers.fc(input=images, size=128, act='relu', program=program) - hidden2 = layers.fc(input=hidden1, size=64, act='relu', program=program) + name='label', shape=[1], data_type='int32', main_program=program) + hidden1 = layers.fc(input=images, + size=128, + act='relu', + main_program=program) + hidden2 = layers.fc(input=hidden1, + size=64, + act='relu', + main_program=program) predict = layers.fc(input=hidden2, size=10, act='softmax', - program=program) - cost = layers.cross_entropy(input=predict, label=label, program=program) - avg_cost = layers.mean(x=cost, program=program) + main_program=program) + cost = layers.cross_entropy( + input=predict, label=label, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) print str(program) def test_simple_conv2d(self): program = Program() images = layers.data( - name='pixel', shape=[3, 48, 48], data_type='int32', program=program) + name='pixel', + shape=[3, 48, 48], + data_type='int32', + main_program=program) layers.conv2d( - input=images, num_filters=3, filter_size=[4, 4], program=program) + input=images, + num_filters=3, + filter_size=[4, 4], + main_program=program) print str(program) @@ -57,9 +73,9 @@ class TestBook(unittest.TestCase): name='pixel', shape=[1, 28, 28], data_type='float32', - program=program) + main_program=program) label = layers.data( - name='label', shape=[1], data_type='int32', program=program) + name='label', shape=[1], data_type='int32', main_program=program) conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, @@ -67,7 +83,7 @@ class TestBook(unittest.TestCase): pool_size=2, pool_stride=2, act="relu", - program=program) + main_program=program) conv_pool_2 = nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, @@ -75,14 +91,15 @@ class TestBook(unittest.TestCase): pool_size=2, pool_stride=2, act="relu", - program=program) + main_program=program) predict = layers.fc(input=conv_pool_2, size=10, act="softmax", - program=program) - cost = layers.cross_entropy(input=predict, label=label, program=program) - avg_cost = layers.mean(x=cost, program=program) + main_program=program) + cost = layers.cross_entropy( + input=predict, label=label, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) program.append_backward(avg_cost) @@ -93,58 +110,58 @@ class TestBook(unittest.TestCase): dict_size = 10000 embed_size = 32 first_word = layers.data( - name='firstw', shape=[1], data_type='int64', program=program) + name='firstw', shape=[1], data_type='int64', main_program=program) second_word = layers.data( - name='secondw', shape=[1], data_type='int64', program=program) + name='secondw', shape=[1], data_type='int64', main_program=program) third_word = layers.data( - name='thirdw', shape=[1], data_type='int64', program=program) + name='thirdw', shape=[1], data_type='int64', main_program=program) forth_word = layers.data( - name='forthw', shape=[1], data_type='int64', program=program) + name='forthw', shape=[1], data_type='int64', main_program=program) next_word = layers.data( - name='nextw', shape=[1], data_type='int64', program=program) + name='nextw', shape=[1], data_type='int64', main_program=program) embed_first = layers.embedding( input=first_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_second = layers.embedding( input=second_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_third = layers.embedding( input=third_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) embed_forth = layers.embedding( input=forth_word, size=[dict_size, embed_size], data_type='float32', param_attr={'name': 'shared_w'}, - program=program) + main_program=program) concat_embed = layers.concat( input=[embed_first, embed_second, embed_third, embed_forth], axis=1, - program=program) + main_program=program) hidden1 = layers.fc(input=concat_embed, size=256, act='sigmoid', - program=program) + main_program=program) predict_word = layers.fc(input=hidden1, size=dict_size, act='softmax', - program=program) + main_program=program) cost = layers.cross_entropy( - input=predict_word, label=next_word, program=program) - avg_cost = layers.mean(x=cost, program=program) + input=predict_word, label=next_word, main_program=program) + avg_cost = layers.mean(x=cost, main_program=program) self.assertIsNotNone(avg_cost) print str(program) diff --git a/python/paddle/v2/framework/tests/test_lod_array_length_op.py b/python/paddle/v2/framework/tests/test_lod_array_length_op.py new file mode 100644 index 0000000000000000000000000000000000000000..af2b4d705e7ec121bd5f1350f0a642ae8c44bf1e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_array_length_op.py @@ -0,0 +1,21 @@ +import unittest +import paddle.v2.framework.layers as layers +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.core as core +import numpy + + +class TestLoDArrayLength(unittest.TestCase): + def test_array_length(self): + tmp = layers.zeros(shape=[10], dtype='int32') + i = layers.fill_constant(shape=[1], dtype='int64', value=10) + arr = layers.array_write(tmp, i=i) + arr_len = layers.array_length(arr) + cpu = core.CPUPlace() + exe = Executor(cpu) + result = numpy.array(exe.run(fetch_list=[arr_len])[0]) + self.assertEqual(11, result[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_rank_table.py b/python/paddle/v2/framework/tests/test_lod_rank_table.py new file mode 100644 index 0000000000000000000000000000000000000000..408145c10f46e24e8a54b05b4f3afa9231b6ffd6 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_rank_table.py @@ -0,0 +1,28 @@ +from paddle.v2.framework.layers import lod_rank_table, data +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.framework import g_main_program +import paddle.v2.framework.core as core +import numpy +import unittest + + +class TestLoDRankTable(unittest.TestCase): + def test_lod_rank_table(self): + x = data(name='x', shape=[100]) + cpu = core.CPUPlace() + rank_table = lod_rank_table(x=x, level=1) + rank_table.persistable = True + exe = Executor(cpu) + scope = core.Scope() + + tensor = core.LoDTensor() + tensor.set(numpy.random.random(size=(17, 100)), cpu) + tensor.set_lod([[0, 1, 3], [0, 5, 6, 7], [0, 3, 4, 9, 10, 13, 16, 17]]) + exe.run(g_main_program, scope=scope, feed={'x': tensor}) + var = scope.find_var(rank_table.name) + table = var.get_lod_rank_table() + self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_reset_op.py b/python/paddle/v2/framework/tests/test_lod_reset_op.py new file mode 100644 index 0000000000000000000000000000000000000000..652ccecfa443fc95f08f52df766709cb550f4049 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_reset_op.py @@ -0,0 +1,64 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestLodResetOpByAttr(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0 = [0, 7, 10] + self.inputs = {'X': (x, lod)} + self.attrs = {'target_lod': target_lod_0} + self.outputs = {'Out': (x, [target_lod_0])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out") + + +class TestLodResetOpByInput(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0 = [0, 4, 7, 10] + self.inputs = { + 'X': (x, lod), + 'TargetLoD': np.array([target_lod_0]).astype('int32') + } + self.outputs = {'Out': (x, [target_lod_0])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out", no_grad_set=set("TargetLoD")) + + +class TestLodResetOpBoth(OpTest): + def setUp(self): + self.op_type = "lod_reset" + x = np.random.random((10, 20)).astype("float32") + lod = [[0, 3, 5, 10]] + target_lod_0_attr = [0, 7, 10] + target_lod_0_in = [0, 4, 7, 10] + self.inputs = { + 'X': (x, lod), + 'TargetLoD': np.array(target_lod_0_in).astype('int32') + } + self.attrs = {'target_lod': target_lod_0_attr} + self.outputs = {'Out': (x, [target_lod_0_in])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Out", no_grad_set=set("TargetLoD")) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_tensor_array.py b/python/paddle/v2/framework/tests/test_lod_tensor_array.py new file mode 100644 index 0000000000000000000000000000000000000000..a433bcf622b14a1d2d33b5b98d555e1a21e4b9e8 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_tensor_array.py @@ -0,0 +1,38 @@ +import unittest +import paddle.v2.framework.core as core +import numpy + + +class TestLoDTensorArray(unittest.TestCase): + def test_get_set(self): + scope = core.Scope() + arr = scope.var('tmp_lod_tensor_array') + tensor_array = arr.get_lod_tensor_array() + self.assertEqual(0, len(tensor_array)) + cpu = core.CPUPlace() + for i in xrange(10): + t = core.LoDTensor() + t.set(numpy.array([i], dtype='float32'), cpu) + t.set_lod([[0, 1]]) + tensor_array.append(t) + + self.assertEqual(10, len(tensor_array)) + + for i in xrange(10): + t = tensor_array[i] + self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32')) + self.assertEqual([[0, 1]], t.lod()) + + t = core.LoDTensor() + t.set(numpy.array([i + 10], dtype='float32'), cpu) + t.set_lod([[0, 2]]) + tensor_array[i] = t + t = tensor_array[i] + self.assertEqual( + numpy.array(t), numpy.array( + [i + 10], dtype='float32')) + self.assertEqual([[0, 2]], t.lod()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py b/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..e9713666b3f64d7a39afadab7da6b22f149b8cf8 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_lod_tensor_array_ops.py @@ -0,0 +1,165 @@ +import unittest +import paddle.v2.framework.core as core +import numpy +import paddle.v2.framework.layers as layers +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops + + +class TestCPULoDTensorArrayOps(unittest.TestCase): + def place(self): + return core.CPUPlace() + + def test_lod_tensor_to_array_level_0(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 3, 9, 10]]) + expect = map(lambda x: numpy.array(x).astype('int32'), + [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) + self.main(tensor=tensor, expect_array=expect, expect_lod=[] * 6) + + def test_lod_tensor_to_array_level_0_empty_seq(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 3, 9, 9, 10]]) + expect = map(lambda x: numpy.array(x).astype('int32'), + [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) + self.main(tensor=tensor, expect_array=expect, expect_lod=[] * 6) + + def test_lod_tensor_to_array_level_1(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(20).reshape(20, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 2, 5], [0, 3, 9, 11, 17, 20]]) + + expect = [ + numpy.array( + [9, 10, 0, 1, 2], dtype='int32'), numpy.array( + [11, 12, 13, 14, 15, 16, 3, 4, 5, 6, 7, 8], dtype='int32'), + numpy.array( + [17, 18, 19], dtype='int32') + ] + + lod = [[[0, 2, 5]], [[0, 6, 12]], [[0, 3]]] + self.main(tensor=tensor, expect_array=expect, expect_lod=lod) + + def test_lod_tensor_to_array_level_1_empty_seq(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(31).reshape(31, 1).astype('int32'), self.place()) + + tensor.set_lod([[0, 3, 5, 9, 11], + [0, 3, 7, 11, 11, 12, 17, 19, 21, 23, 30, 31]]) + + expect = [ + numpy.array( + item, dtype='int32') + for item in [[ + 12, 13, 14, 15, 16, 0, 1, 2, 23, 24, 25, 26, 27, 28, 29 + ], [17, 18, 3, 4, 5, 6, 11, 30], [19, 20, 7, 8, 9, 10], [21, 22]] + ] + + lod = [[[0, 5, 8, 8, 15]], [[0, 2, 6, 7, 8]], [[0, 2, 6]], [[0, 2]]] + self.main(tensor=tensor, expect_array=expect, expect_lod=lod) + + def test_lod_tensor_to_array_level_2(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(50).reshape(50, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 2, 5, 6], [0, 2, 5, 6, 10, 12, 13], + [0, 3, 7, 11, 17, 21, 22, 23, 27, 31, 39, 45, 46, 50]]) + + expect = [ + numpy.array( + item, dtype='int32') + for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], range( + 22, 39) + range(7, 21), range(39, 46)] + ] + lod = [[[0, 1, 3, 4], [0, 1, 4, 8, 12]], + [[0, 4, 7], [0, 1, 5, 9, 17, 21, 27, 31]], [[0, 2], [0, 6, 7]]] + self.main(tensor=tensor, expect_array=expect, expect_lod=lod) + + def test_lod_tensor_to_array_level_2_skip_level(self): + tensor = core.LoDTensor() + tensor.set( + numpy.arange(50).reshape(50, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 2, 5, 6], [0, 2, 5, 6, 10, 12, 13], + [0, 3, 7, 11, 17, 21, 22, 23, 27, 31, 39, 45, 46, 50]]) + self.main(tensor=tensor, expect_array=None, expect_lod=None, level=1) + + def main(self, tensor, expect_array, expect_lod, level=0): + place = self.place() + program = Program() + x = layers.data(name='x', shape=[10], main_program=program) + x.persistable = True + table = layers.lod_rank_table(x, level=level, main_program=program) + array = layers.lod_tensor_to_array(x, table, main_program=program) + array.persistable = True + + result = layers.array_to_lod_tensor(array, table, main_program=program) + result.persistable = True + exe = Executor(place) + scope = core.Scope() + exe.run(program, feed={'x': tensor}, scope=scope) + var = scope.find_var(array.name) + array = var.get_lod_tensor_array() + if expect_array is not None and expect_lod is not None: + self.check_array_same(array, expect_array, expect_lod) + self.check_tensor_same(scope.find_var(result.name).get_tensor(), tensor) + + def check_array_same(self, array, expect_tensor, expect_lod): + self.assertEqual(len(expect_tensor), len(array)) + for i, exp in enumerate(zip(expect_tensor, expect_lod)): + exp_tensor, exp_lod = exp + exp_tensor = numpy.expand_dims(exp_tensor, axis=1) + self.assertTrue(numpy.allclose(exp_tensor, numpy.array(array[i]))) + self.assertEqual(exp_lod, array[i].lod()) + + def check_tensor_same(self, actual, expect): + self.assertTrue( + numpy.allclose(numpy.array(actual), numpy.array(expect))) + self.assertEqual(actual.lod(), expect.lod()) + + +class TestCPULoDTensorArrayOpGrad(unittest.TestCase): + def test_grad(self): + place = core.CPUPlace() + program = Program() + + x = layers.data( + name='x', + shape=[1], + data_type='float32', + main_program=program, + stop_gradient=False) + table = layers.lod_rank_table(x, level=0, main_program=program) + array = layers.lod_tensor_to_array(x, table, main_program=program) + result = layers.array_to_lod_tensor(array, table, main_program=program) + + mean = layers.mean(x=result, main_program=program) + + append_backward_ops(mean) + + tensor = core.LoDTensor() + tensor.set(numpy.arange(10).reshape(10, 1).astype('float32'), place) + tensor.set_lod([[0, 3, 9, 10]]) + + g_vars = program.global_block().var(x.name + "@GRAD") + + exe = Executor(place) + g_out = [ + item.sum() + for item in map( + numpy.array, + exe.run(program, feed={'x': tensor}, fetch_list=[g_vars])) + ] + g_out_sum = numpy.array(g_out).sum() + + self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_lstm_op.py b/python/paddle/v2/framework/tests/test_lstm_op.py index ff75160083f2936dd653a8396254bf16d1752ffa..77f062e8c8870ec9cc56c9566108abe74665ae30 100644 --- a/python/paddle/v2/framework/tests/test_lstm_op.py +++ b/python/paddle/v2/framework/tests/test_lstm_op.py @@ -117,8 +117,9 @@ class TestLstmOp(OpTest): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = True + self.has_initial_state = False self.is_reverse = False + self.use_peepholes = True def setUp(self): self.set_argument() @@ -128,18 +129,28 @@ class TestLstmOp(OpTest): N = len(self.lod[0]) - 1 x = np.random.normal(size=(T, 4 * self.D)).astype('float64') - h0 = np.zeros((N, self.D)).astype('float64') - c0 = np.zeros((N, self.D)).astype('float64') + if self.has_initial_state: + h0 = np.random.normal(size=(N, self.D)).astype('float64') + c0 = np.random.normal(size=(N, self.D)).astype('float64') + else: + h0 = np.zeros((N, self.D)).astype('float64') + c0 = np.zeros((N, self.D)).astype('float64') w = np.random.normal(size=(self.D, 4 * self.D)).astype('float64') - b = np.random.normal(size=(1, 7 * self.D)).astype('float64') + if self.use_peepholes: + b = np.random.normal(size=(1, 7 * self.D)).astype('float64') + else: + b = np.random.normal(size=(1, 4 * self.D)).astype('float64') w_b = b[:, 0:4 * self.D] - w_c = b[:, 4 * self.D:] + w_c = b[:, 4 * self.D:] if self.use_peepholes else None h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse, ACTVATION[self.act_gate], ACTVATION[self.act_cell], ACTVATION[self.act_cand]) - self.inputs = {'Input': (x, self.lod), 'Weight': w, 'Bias': b} + self.inputs = {'Input': (x, self.lod), 'Weight': w} + + self.inputs['Bias'] = b + if self.has_initial_state: self.inputs['H0'] = h0 self.inputs['C0'] = c0 @@ -149,17 +160,16 @@ class TestLstmOp(OpTest): 'Cell': (c, self.lod), } self.attrs = { - 'usePeepholes': True, - 'isReverse': self.is_reverse, - 'gateActivation': self.act_gate, - 'cellActivation': self.act_cell, - 'candidateActivation': self.act_cand + 'use_peepholes': self.use_peepholes, + 'is_reverse': self.is_reverse, + 'gate_activation': self.act_gate, + 'cell_activation': self.act_cell, + 'candidate_activation': self.act_cand } def test_check_output(self): self.check_output(atol=1e-8) - #TODO(qingqing) add more unit testing case def test_check_grad(self): # TODO(qingqing) remove folowing lines after the check_grad is refined. N = len(self.lod[0]) - 1 @@ -170,7 +180,7 @@ class TestLstmOp(OpTest): ['Input', 'Weight', 'Bias'], ['Hidden'], max_relative_error=5e-4) -class TestLstmOpHasNoInitial(TestLstmOp): +class TestLstmOpHasInitial(TestLstmOp): def set_argument(self): self.lod = [[0, 2, 5, 7]] self.D = 16 @@ -179,8 +189,69 @@ class TestLstmOpHasNoInitial(TestLstmOp): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = False + self.has_initial_state = True self.is_reverse = True + self.use_peepholes = True + + def test_check_grad(self): + # TODO(qingqing) remove folowing lines after the check_grad is refined. + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'H0', 'C0'], ['Hidden'], + max_relative_error=5e-4) + + def test_check_grad_ingore_bias(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Bias')) + + def test_check_grad_ingore_weight(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Bias'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Weight')) + + def test_check_grad_ingore_input(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Weight', 'Bias'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('Input')) + + def test_check_grad_ingore_h0(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'C0'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('H0')) + + def test_check_grad_ingore_c0(self): + N = len(self.lod[0]) - 1 + self.outputs['BatchGate'] = np.zeros((N, 4 * self.D)).astype('float64') + self.outputs['BatchCellPreAct'] = np.zeros( + (N, self.D)).astype('float64') + self.check_grad( + ['Input', 'Weight', 'Bias', 'H0'], ['Hidden'], + max_relative_error=5e-4, + no_grad_set=set('C0')) class TestLstmOpRerverse(TestLstmOp): @@ -192,8 +263,23 @@ class TestLstmOpRerverse(TestLstmOp): self.act_cell = 'tanh' self.act_cand = 'tanh' - self.has_initial_state = True + self.has_initial_state = False + self.is_reverse = True + self.use_peepholes = True + + +class TestLstmOpNotUsePeepholes(TestLstmOp): + def set_argument(self): + self.lod = [[0, 2, 5, 7]] + self.D = 16 + + self.act_gate = 'sigmoid' + self.act_cell = 'tanh' + self.act_cand = 'tanh' + + self.has_initial_state = False self.is_reverse = True + self.use_peepholes = False if __name__ == '__main__': diff --git a/python/paddle/v2/framework/tests/test_momentum_op.py b/python/paddle/v2/framework/tests/test_momentum_op.py index 654d31975aab4578055e7e70ade202bd2c3d93cb..638095f7564c8761151a7794f98f9ca797b0083b 100644 --- a/python/paddle/v2/framework/tests/test_momentum_op.py +++ b/python/paddle/v2/framework/tests/test_momentum_op.py @@ -37,7 +37,7 @@ class TestMomentumOp1(OpTest): class TestMomentumOp2(OpTest): - '''Test Momentum with defaukt values for attributes + '''Test Momentum with default values for attributes ''' def setUp(self): @@ -57,7 +57,7 @@ class TestMomentumOp2(OpTest): 'LearningRate': learning_rate } - self.attrs = {'mu': mu, 'useNesterov': use_nesterov} + self.attrs = {'mu': mu, 'use_nesterov': use_nesterov} velocity_out = mu * velocity + grad if use_nesterov: diff --git a/python/paddle/v2/framework/tests/test_operator_desc.py b/python/paddle/v2/framework/tests/test_operator_desc.py index 7355f72455ca4f821c9520d97162e3e0050383af..a0bc4e0b91602cfc90f91a1e2dd4bce22c0dbf6d 100644 --- a/python/paddle/v2/framework/tests/test_operator_desc.py +++ b/python/paddle/v2/framework/tests/test_operator_desc.py @@ -1,11 +1,11 @@ import unittest -from paddle.v2.framework.framework import Variable, Program, g_program +from paddle.v2.framework.framework import Variable, Program, g_main_program import paddle.v2.framework.core as core class TestOperator(unittest.TestCase): def test_error_type(self): - block = g_program.create_block() + block = g_main_program.create_block() try: block.append_op() self.assertFail() diff --git a/python/paddle/v2/framework/tests/test_optimizer.py b/python/paddle/v2/framework/tests/test_optimizer.py index 45396c9bec9ccf0668b048b2b4855d7a665ebea5..a39e7402600c7a94301de030c90ea51264248cf1 100644 --- a/python/paddle/v2/framework/tests/test_optimizer.py +++ b/python/paddle/v2/framework/tests/test_optimizer.py @@ -7,6 +7,7 @@ from paddle.v2.framework.backward import append_backward_ops class TestOptimizer(unittest.TestCase): def test_sgd_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -22,12 +23,13 @@ class TestOptimizer(unittest.TestCase): outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.01) - opts = sgd_optimizer.minimize(mul_out) + opts = sgd_optimizer.minimize(mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") def test_sgd_optimizer_with_global_step(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -44,15 +46,22 @@ class TestOptimizer(unittest.TestCase): attrs={"x_num_col_dims": 1}) global_step = block.create_var( dtype="float32", shape=[1], lod_level=0, name="step") + learning_rate = 0.01 sgd_optimizer = optimizer.SGDOptimizer( - learning_rate=0.01, global_step=global_step) - opts = sgd_optimizer.minimize(mul_out) + learning_rate=learning_rate, global_step=global_step) + opts = sgd_optimizer.minimize(mul_out, init_program) self.assertEqual(len(opts), 2) sgd_op = opts[0] self.assertEqual(sgd_op.type, "sgd") increment_op = opts[1] self.assertEqual(increment_op.type, "increment") + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 1) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + class TestMomentumOptimizer(unittest.TestCase): class MockMomentum(optimizer.MomentumOptimizer): @@ -63,6 +72,7 @@ class TestMomentumOptimizer(unittest.TestCase): return self._velocity_acc_str def test_vanilla_momentum_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -77,16 +87,18 @@ class TestMomentumOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) - momentum_optimizer = self.MockMomentum(learning_rate=0.01, momentum=0.2) + learning_rate = 0.01 + momentum_optimizer = self.MockMomentum( + learning_rate=learning_rate, momentum=0.2) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) - opts = momentum_optimizer.create_optimization_pass(params_grads, - mul_out) + opts = momentum_optimizer.create_optimization_pass( + params_grads, mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") - self.assertFalse(sgd_op.attr('useNesterov')) + self.assertFalse(sgd_op.attr('use_nesterov')) # Check accumulators accumulators = momentum_optimizer.get_accumulators() @@ -96,7 +108,16 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + def test_nesterov_momentum_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -111,17 +132,18 @@ class TestMomentumOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 momentum_optimizer = self.MockMomentum( - learning_rate=0.01, momentum=0.2, use_nesterov=True) + learning_rate=learning_rate, momentum=0.2, use_nesterov=True) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(momentum_optimizer.get_accumulators()), 0) - opts = momentum_optimizer.create_optimization_pass(params_grads, - mul_out) + opts = momentum_optimizer.create_optimization_pass( + params_grads, mul_out, init_program) self.assertEqual(len(opts), 1) sgd_op = opts[0] self.assertEqual(sgd_op.type, "momentum") - self.assertTrue(sgd_op.attr('useNesterov')) + self.assertTrue(sgd_op.attr('use_nesterov')) # Check accumulators accumulators = momentum_optimizer.get_accumulators() @@ -131,6 +153,14 @@ class TestMomentumOptimizer(unittest.TestCase): self.assertEqual(len(velocity_acc), 1) self.assertTrue(mul_x.name in velocity_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + class TestAdagradOptimizer(unittest.TestCase): class MockAdagrad(optimizer.AdagradOptimizer): @@ -141,6 +171,7 @@ class TestAdagradOptimizer(unittest.TestCase): return self._moment_acc_str def test_adagrad_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -155,11 +186,14 @@ class TestAdagradOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) - adagrad_optimizer = self.MockAdagrad(learning_rate=0.01, epsilon=1.0e-6) + learning_rate = 0.01 + adagrad_optimizer = self.MockAdagrad( + learning_rate=learning_rate, epsilon=1.0e-6) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adagrad_optimizer.get_accumulators()), 0) - opts = adagrad_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adagrad_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 1) adagrad_op = opts[0] self.assertEqual(adagrad_op.type, "adagrad") @@ -172,6 +206,14 @@ class TestAdagradOptimizer(unittest.TestCase): self.assertEqual(len(moment_acc), 1) self.assertTrue(mul_x.name in moment_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 2) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + self.assertEqual(init_ops[1].type, "fill_constant") + self.assertAlmostEqual(init_ops[1].attr('value'), 0.0) + class TestAdamOptimizer(unittest.TestCase): class MockAdam(optimizer.AdamOptimizer): @@ -185,6 +227,7 @@ class TestAdamOptimizer(unittest.TestCase): return self._moment2_acc_str def test_adam_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -199,12 +242,14 @@ class TestAdamOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 adam_optimizer = self.MockAdam( - learning_rate=0.01, beta1=0.9, beta2=0.999) + learning_rate=learning_rate, beta1=0.9, beta2=0.999) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adam_optimizer.get_accumulators()), 0) - opts = adam_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adam_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 3) adam_op = opts[0] self.assertEqual(adam_op.type, "adam") @@ -221,6 +266,12 @@ class TestAdamOptimizer(unittest.TestCase): self.assertTrue(mul_x.name in moment1_acc) self.assertTrue(mul_x.name in moment2_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 5) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + class TestAdamaxOptimizer(unittest.TestCase): class MockAdamax(optimizer.AdamaxOptimizer): @@ -234,6 +285,7 @@ class TestAdamaxOptimizer(unittest.TestCase): return self._inf_norm_acc_str def test_adamax_optimizer(self): + init_program = framework.Program() program = framework.Program() block = program.global_block() mul_x = block.create_parameter( @@ -248,12 +300,14 @@ class TestAdamaxOptimizer(unittest.TestCase): "Y": mul_y}, outputs={"Out": mul_out}, attrs={"x_num_col_dims": 1}) + learning_rate = 0.01 adamax_optimizer = self.MockAdamax( - learning_rate=0.01, beta1=0.9, beta2=0.999) + learning_rate=learning_rate, beta1=0.9, beta2=0.999) params_grads = append_backward_ops(mul_out) self.assertEqual(len(params_grads), 1) self.assertEqual(len(adamax_optimizer.get_accumulators()), 0) - opts = adamax_optimizer.create_optimization_pass(params_grads, mul_out) + opts = adamax_optimizer.create_optimization_pass(params_grads, mul_out, + init_program) self.assertEqual(len(opts), 2) adam_op = opts[0] self.assertEqual(adam_op.type, "adamax") @@ -270,6 +324,12 @@ class TestAdamaxOptimizer(unittest.TestCase): self.assertTrue(mul_x.name in moment_acc) self.assertTrue(mul_x.name in inf_norm_acc) + # Check init_program + init_ops = init_program.global_block().ops + self.assertEqual(len(init_ops), 4) + self.assertEqual(init_ops[0].type, "fill_constant") + self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_parameter.py b/python/paddle/v2/framework/tests/test_parameter.py index 1ac0cdd99f1b7c15d64ae9d2c465d5a9d563bd80..f04eb4cf27276b0f7da0793c97742ac42e4583be 100644 --- a/python/paddle/v2/framework/tests/test_parameter.py +++ b/python/paddle/v2/framework/tests/test_parameter.py @@ -1,11 +1,11 @@ import unittest -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program import paddle.v2.framework.core as core class TestParameter(unittest.TestCase): def test_param(self): - b = g_program.create_block() + b = g_main_program.create_block() param = b.create_parameter( name='fc.w', shape=[784, 100], diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index c93469e11994c44ee6fbd1a8828074c1558c08fa..ac3fa6aa87835b3cd6fb9bbf6fe66b1d0c577ca2 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -61,8 +61,8 @@ class TestPool2d_Op(OpTest): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'poolingType': self.pool_type, - 'globalPooling': self.global_pool, + 'pooling_type': self.pool_type, + 'global_pooling': self.global_pool, } self.outputs = {'Out': output.astype('float32')} diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index 416f0df7cd27f58c4c99fb776b84e44005f31639..87483ae5e568c01141ff789f37e84069cb8e827d 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -67,8 +67,8 @@ class TestPool3d_Op(OpTest): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'poolingType': self.pool_type, - 'globalPooling': self.global_pool, + 'pooling_type': self.pool_type, + 'global_pooling': self.global_pool, } self.outputs = {'Out': output.astype('float32')} diff --git a/python/paddle/v2/framework/tests/test_pool_max_op.py b/python/paddle/v2/framework/tests/test_pool_max_op.py index cc1a867761142edea506a24e84ad31bfe6858fb0..04843a28ac19e076e097d1aa1034bcf9378aa495 100644 --- a/python/paddle/v2/framework/tests/test_pool_max_op.py +++ b/python/paddle/v2/framework/tests/test_pool_max_op.py @@ -86,7 +86,7 @@ class TestMaxPoolWithIndex_Op(OpTest): 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'globalPooling': self.global_pool, + 'global_pooling': self.global_pool, } self.inputs = {'X': input} diff --git a/python/paddle/v2/framework/tests/test_positive_negative_pair_op.py b/python/paddle/v2/framework/tests/test_positive_negative_pair_op.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a6c428a26dece01fe2958991edd3edf3a8266e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_positive_negative_pair_op.py @@ -0,0 +1,106 @@ +import unittest +import itertools +import numpy as np +from op_test import OpTest + + +def py_pnpair_op(score, label, query, column=-1, weight=None): + # group by query id + predictions = {} + batch_size = label.shape[0] + if weight is None: + weight = np.ones(shape=(batch_size, 1)).astype('float32') + for s, l, q, w in zip(score, label, query, weight): + s, l, q, w = s[column], l[0], q[0], w[0] + if q not in predictions: + predictions[q] = [] + predictions[q].append((s, l, w)) + + # accumulate statistics + pos, neg, neu = 0, 0, 0 + for _, ranks in predictions.items(): + for e1, e2 in itertools.combinations(ranks, 2): + s1, s2, l1, l2, w1, w2 = e1[0], e2[0], e1[1], e2[1], e1[2], e2[2] + w = (w1 + w2) * 0.5 + if l1 == l2: + continue + if s1 == s2: + neu += w + elif (s1 - s2) * (l1 - l2) > 0: + pos += w + else: + neg += w + + return np.array(pos).astype('float32'), np.array(neg).astype( + 'float32'), np.array(neu).astype('float32') + + +class TestPositiveNegativePairOp(OpTest): + def setUp(self): + self.op_type = 'positive_negative_pair' + batch_size = 20 + max_query_id = 5 + score = np.random.normal(size=(batch_size, 1)).astype('float32') + label = np.random.normal(size=(batch_size, 1)).astype('float32') + query = np.array( + [np.random.randint(max_query_id) for i in range(batch_size)]) + query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') + + pos, neg, neu = py_pnpair_op(score, label, query) + self.inputs = {'Score': score, 'Label': label, 'QueryID': query} + self.attrs = {'column': -1} + self.outputs = { + 'PositivePair': pos, + 'NegativePair': neg, + 'NeutralPair': neu + } + + def test_check_output(self): + self.check_output() + + +class TestPositiveNegativePairOpAccumulateWeight(OpTest): + def setUp(self): + self.op_type = 'positive_negative_pair' + batch_size = 20 + max_query_id = 5 + max_random_num = 2 << 15 + score_dim = 2 + score = np.random.normal(size=(batch_size, 2)).astype('float32') + label = np.random.normal(size=(batch_size, 1)).astype('float32') + weight = np.random.normal(size=(batch_size, 1)).astype('float32') + query = np.array( + [np.random.randint(max_query_id) for i in range(batch_size)]) + query = np.reshape(query, newshape=(batch_size, 1)).astype('int64') + acc_pos = np.reshape( + np.random.randint(max_random_num), newshape=(1)).astype('float32') + acc_neg = np.reshape( + np.random.randint(max_random_num), newshape=(1)).astype('float32') + acc_neu = np.reshape( + np.random.randint(max_random_num), newshape=(1)).astype('float32') + column = np.random.randint(score_dim) + + pos, neg, neu = py_pnpair_op( + score, label, query, column=column, weight=weight) + self.inputs = { + 'Score': score, + 'Label': label, + 'QueryID': query, + 'AccumulatePositivePair': acc_pos, + 'AccumulateNegativePair': acc_neg, + 'AccumulateNeutralPair': acc_neu, + 'Weight': weight + } + self.attrs = {'column': column} + self.outputs = { + 'PositivePair': pos + acc_pos, + 'NegativePair': neg + acc_neg, + 'NeutralPair': neu + acc_neu + } + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_precision_recall_op.py b/python/paddle/v2/framework/tests/test_precision_recall_op.py new file mode 100644 index 0000000000000000000000000000000000000000..d3dbdb6e2aba6dfe98440ad07083cf1ffda5b668 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_precision_recall_op.py @@ -0,0 +1,173 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def calc_precision(tp_count, fp_count): + if tp_count > 0.0 or fp_count > 0.0: + return tp_count / (tp_count + fp_count) + return 1.0 + + +def calc_recall(tp_count, fn_count): + if tp_count > 0.0 or fn_count > 0.0: + return tp_count / (tp_count + fn_count) + return 1.0 + + +def calc_f1_score(precision, recall): + if precision > 0.0 or recall > 0.0: + return 2 * precision * recall / (precision + recall) + return 0.0 + + +def get_states(idxs, labels, cls_num, weights=None): + ins_num = idxs.shape[0] + # TP FP TN FN + states = np.zeros((cls_num, 4)).astype('float32') + for i in xrange(ins_num): + w = weights[i] if weights is not None else 1.0 + idx = idxs[i][0] + label = labels[i][0] + if idx == label: + states[idx][0] += w + for j in xrange(cls_num): + states[j][2] += w + states[idx][2] -= w + else: + states[label][3] += w + states[idx][1] += w + for j in xrange(cls_num): + states[j][2] += w + states[label][2] -= w + states[idx][2] -= w + return states + + +def compute_metrics(states, cls_num): + total_tp_count = 0.0 + total_fp_count = 0.0 + total_fn_count = 0.0 + macro_avg_precision = 0.0 + macro_avg_recall = 0.0 + for i in xrange(cls_num): + total_tp_count += states[i][0] + total_fp_count += states[i][1] + total_fn_count += states[i][3] + macro_avg_precision += calc_precision(states[i][0], states[i][1]) + macro_avg_recall += calc_recall(states[i][0], states[i][3]) + metrics = [] + macro_avg_precision /= cls_num + macro_avg_recall /= cls_num + metrics.append(macro_avg_precision) + metrics.append(macro_avg_recall) + metrics.append(calc_f1_score(macro_avg_precision, macro_avg_recall)) + micro_avg_precision = calc_precision(total_tp_count, total_fp_count) + metrics.append(micro_avg_precision) + micro_avg_recall = calc_recall(total_tp_count, total_fn_count) + metrics.append(micro_avg_recall) + metrics.append(calc_f1_score(micro_avg_precision, micro_avg_recall)) + return np.array(metrics).astype('float32') + + +class TestPrecisionRecallOp_0(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + states = get_states(idxs, labels, cls_num) + metrics = compute_metrics(states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = {'MaxProbs': max_probs, 'Indices': idxs, 'Labels': labels} + + self.outputs = { + 'BatchMetrics': metrics, + 'AccumMetrics': metrics, + 'AccumStatesInfo': states + } + + def test_check_output(self): + self.check_output() + + +class TestPrecisionRecallOp_1(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + + states = get_states(idxs, labels, cls_num, weights) + metrics = compute_metrics(states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = { + 'MaxProbs': max_probs, + 'Indices': idxs, + 'Labels': labels, + 'Weights': weights + } + + self.outputs = { + 'BatchMetrics': metrics, + 'AccumMetrics': metrics, + 'AccumStatesInfo': states + } + + def test_check_output(self): + self.check_output() + + +class TestPrecisionRecallOp_2(OpTest): + def setUp(self): + self.op_type = "precision_recall" + ins_num = 64 + cls_num = 10 + max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') + labels = np.random.choice(xrange(cls_num), ins_num).reshape( + (ins_num, 1)).astype('int32') + states = np.random.randint(0, 30, (cls_num, 4)).astype('float32') + + accum_states = get_states(idxs, labels, cls_num, weights) + batch_metrics = compute_metrics(accum_states, cls_num) + accum_states += states + accum_metrics = compute_metrics(accum_states, cls_num) + + self.attrs = {'class_number': cls_num} + + self.inputs = { + 'MaxProbs': max_probs, + 'Indices': idxs, + 'Labels': labels, + 'Weights': weights, + 'StatesInfo': states + } + + self.outputs = { + 'BatchMetrics': batch_metrics, + 'AccumMetrics': accum_metrics, + 'AccumStatesInfo': accum_states + } + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_program.py b/python/paddle/v2/framework/tests/test_program.py index be020573b7dcd9f8dcd0f99d654dc8b2106abb2b..7be67b6614ee3302a319289b821a214a81b6f64e 100644 --- a/python/paddle/v2/framework/tests/test_program.py +++ b/python/paddle/v2/framework/tests/test_program.py @@ -2,35 +2,35 @@ import unittest import paddle.v2.framework.core as core from paddle.v2.framework.framework import Program -from paddle.v2.framework.framework import g_program +from paddle.v2.framework.framework import g_main_program class TestProgram(unittest.TestCase): def test_program(self): - b = g_program.current_block() + b = g_main_program.current_block() self.assertEqual(-1, b.parent_idx) self.assertEqual(0, b.idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(2, b.idx) self.assertEqual(1, b.parent_idx) - g_program.rollback() + g_main_program.rollback() - b = g_program.current_block() + b = g_main_program.current_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) - b = g_program.create_block() + b = g_main_program.create_block() self.assertEqual(3, b.idx) self.assertEqual(1, b.parent_idx) - g_program.rollback() - b = g_program.current_block() + g_main_program.rollback() + b = g_main_program.current_block() self.assertEqual(1, b.idx) self.assertEqual(0, b.parent_idx) diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 6c9081a7c37d2a68c50b5748c87199efe9a90cc7..16100429dd4010eb5c9a3e8896212f39295a4c8a 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -1,51 +1,64 @@ -import logging -import paddle.v2.framework.core as core import unittest + +import paddle.v2.framework.layers as layers +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops import numpy as np -from paddle.v2.framework.op import Operator, RecurrentOp -from op_test import get_numeric_gradient +import paddle.v2.framework.core as core -def py_sigmoid(x): - return 1. / (1. + np.exp(-x)) +class PyRNNBase(object): + def __init__(self, input_shape, output_shape): + self.x = np.ones(shape=input_shape).astype("float32") + self.y = np.zeros(shape=output_shape).astype("float32") + def step(self, step_id, x): + raise NotImplementedError -class PySimpleRNN(object): - ''' - A simple implementation of RNN based on numpy, to futhur test RecurrentOp's alogorithm - ''' + def forward(self): + for step_id in range(self.x.shape[0]): + self.step(step_id, self.x[step_id]) + return np.array([np.mean(self.y)]) - def __init__(self, input_dim=30, batch_size=50, weight_dim=15, sent_len=11): - self.x = np.random.normal(size=(sent_len, batch_size, - input_dim)).astype("float32") - self.W = np.random.normal(size=(input_dim, input_dim)).astype("float32") - self.U = np.random.normal(size=(input_dim, input_dim)).astype("float32") + def segment_inputs(self): + return [self.x[i] for i in range(self.x.shape[0])] + + +class PySimpleRNN1(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(PySimpleRNN1, self).__init__(input_shape, output_shape) + + seq_len, batch_size, input_dim = input_shape self.h_boot = np.random.normal(size=(batch_size, input_dim)).astype("float32") - # memories - self.mems = [ - np.zeros(shape=(batch_size, input_dim)).astype("float32") - for i in range(sent_len) - ] + self.scale = 1.0 / 2.0 + men_dim = (seq_len, batch_size, input_dim) + self.mems = np.zeros(shape=men_dim).astype("float32") - def forward(self): - xs = self.segment_inputs() - for step_id in range(self.x.shape[0]): - self.step(step_id, xs[step_id]) - return self.concat_outputs() + def step(self, step_id, x): + if step_id == 0: + pre_mem = self.h_boot + else: + pre_mem = self.mems[step_id - 1] + self.mems[step_id] = (pre_mem + x) * self.scale + self.y[step_id] = self.mems[step_id] - def segment_inputs(self): - return [self.x[i] for i in range(self.x.shape[0])] - def concat_outputs(self): - return np.array(self.mems).astype("float32") +class PySimpleRNN2(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(PySimpleRNN2, self).__init__(input_shape, output_shape) + + seq_len, batch_size, input_dim = input_shape + self.W = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.U = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.h_boot = np.ones(shape=(batch_size, input_dim)).astype("float32") + + men_dim = (seq_len, batch_size, input_dim) + self.mems = np.zeros(shape=men_dim).astype("float32") def step(self, step_id, x): - ''' - run a step - ''' - mem = self.mems[step_id] if step_id > 0: pre_mem = self.mems[step_id - 1] else: @@ -53,108 +66,128 @@ class PySimpleRNN(object): xW = np.matmul(x, self.W).astype("float32") hU = np.matmul(pre_mem, self.U).astype("float32") - sum = xW + hU - self.mems[step_id] = py_sigmoid(sum) - - -class PySimpleRNNTest(unittest.TestCase): - def setUp(self): - self.rnn = PySimpleRNN() + def py_sigmoid(x): + return 1. / (1. + np.exp(-x)) - def test_forward(self): - output = self.rnn.forward() + self.mems[step_id] = py_sigmoid(xW + hU) + self.y[step_id] = self.mems[step_id] -def create_tensor(scope, name, shape, np_data): - tensor = scope.var(name).get_tensor() - tensor.set_dims(shape) - tensor.set(np_data, core.CPUPlace()) +def create_tensor(np_data, place): + tensor = core.LoDTensor() + tensor.set(np_data, place) return tensor -class RecurrentOpTest(unittest.TestCase): +class RecurrentOpTest1(unittest.TestCase): ''' Test RNNOp - equation: - h_t = \sigma (W x_t + U h_{t-1}) - weights: - - W - - U + h_t = ( x_t + h_{t-1} ) / scale vars: - x memories: - h outputs: - - h + - h ''' - input_dim = 30 - batch_size = 50 - weight_dim = 15 - sent_len = 11 + input_dim = 2 + batch_size = 1 + sent_len = 1 + + def setup_program(self): + self.main_program = Program() + self.startup_program = Program() + self.p_info = { + "main_program": self.main_program, + "startup_program": self.startup_program + } + self.place = core.CPUPlace() def setUp(self): - self.py_rnn = PySimpleRNN(self.input_dim, self.batch_size, - self.weight_dim, self.sent_len) + self.setup_program() + self.data_field = {"x", "h_boot"} - def forward(self): - self.scope = core.Scope() - self.create_global_variables() - self.create_rnn_op() - self.create_step_net() - ctx = core.DeviceContext.create(core.CPUPlace()) - self.rnnop.run(self.scope, ctx) - return np.array(self.scope.find_var("h@mem").get_tensor()).astype( - "float32") - - def create_global_variables(self): - # create inlink - x_np_data = self.py_rnn.x - create_tensor(self.scope, "x", - [self.sent_len, self.batch_size, self.input_dim], - x_np_data) - W_np_data = self.py_rnn.W - create_tensor(self.scope, "W", [self.input_dim, self.input_dim], - W_np_data) - - U_np_data = self.py_rnn.U - create_tensor(self.scope, "U", [self.input_dim, self.input_dim], - U_np_data) - - h_boot_np_data = self.py_rnn.h_boot - create_tensor(self.scope, "h_boot", [self.batch_size, self.input_dim], - h_boot_np_data) - self.scope.var("step_scopes") - self.scope.var("h@mem") + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) + + self.output = layers.mean(x=self.create_rnn_op(), **self.p_info) def create_rnn_op(self): - # create RNNOp - self.rnnop = RecurrentOp( - # inputs - inputs=["x"], - initial_states=["h_boot"], - step_net="stepnet", - # outputs - outputs=["h@mem"], - step_scopes="step_scopes", - # attributes - ex_states=["h@pre"], - states=["h@mem"]) - - def create_step_net(self): - stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x", Y="W", Out="Wx") - h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@mem") - - for op in [x_fc_op, h_fc_op, sum_op, sig_op]: - stepnet.append_op(op) - stepnet.complete_add_op(True) - self.rnnop.set_stepnet(stepnet) - - def test_forward(self): + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot = layers.data( + shape=[self.input_dim], + data_type='float32', + name='h_boot', + **self.p_info) + h_boot.stop_gradient = False + + rnn = layers.StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre = rnn.memory(init=h_boot) + x_t = rnn.step_input(x) + + h = layers.scale( + x=layers.elementwise_add( + x=h_pre, y=x_t, **self.p_info), + scale=self.py_rnn.scale, + **self.p_info) + + rnn.update_memory(h_pre, h) + rnn.output(h) + + return rnn() + + def forward(self): + self.feed_map = { + x: create_tensor(getattr(self.py_rnn, x), self.place) + for x in self.data_field + } + exe = Executor(self.place) + out = exe.run(self.main_program, + feed=self.feed_map, + fetch_list=[self.output]) + + return np.array(out[0]) + + def backward(self): + self.feed_map = { + x: create_tensor(getattr(self.py_rnn, x), self.place) + for x in self.data_field + } + fetch_list = [ + self.main_program.global_block().var(x + "@GRAD") + for x in self.data_field + ] + + exe = Executor(self.place) + return exe.run(self.main_program, + feed=self.feed_map, + fetch_list=fetch_list) + + def test_backward(self): + self.check_forward() + + append_backward_ops(self.output) + + ana_grad = [np.array(x) for x in self.backward()] + + num_grad = self.get_numerical_gradient() + for idx, name in enumerate(self.data_field): + self.assertEqual(num_grad[idx].shape, ana_grad[idx].shape) + self.assertTrue( + np.isclose( + num_grad[idx], ana_grad[idx], rtol=0.1).all()) + + def check_forward(self): print 'test recurrent op forward' pd_output = self.forward() py_output = self.py_rnn.forward() @@ -164,44 +197,260 @@ class RecurrentOpTest(unittest.TestCase): self.assertEqual(pd_output.shape, py_output.shape) self.assertTrue(np.isclose(pd_output, py_output, rtol=0.1).all()) + def get_numerical_gradient(self, delta=0.005): + dloss_dout = 1.0 + feed_list = [getattr(self.py_rnn, x) for x in self.data_field] + grad_list = [np.zeros_like(x) for x in feed_list] + for feed, grad in zip(feed_list, grad_list): + for f, g in np.nditer([feed, grad], op_flags=['readwrite']): + o = float(f) + f[...] = o + delta + y_pos = self.forward() + + f[...] = o - delta + y_neg = self.forward() + + f[...] = o + dout_dfeed = (y_pos - y_neg) / (delta * 2) + g[...] = dout_dfeed[0] + + return grad_list + + +class RecurrentOpTest2(RecurrentOpTest1): + ''' + Test RNNOp + equation: + h_t = \sigma (W x_t + U h_{t-1}) + weights: + - W + - U + vars: + - x + memories: + - h + outputs: + - h + ''' + + input_dim = 2 + batch_size = 10 + sent_len = 2 + + def setUp(self): + self.setup_program() + + self.data_field = {"x", "h_boot", "W", "U"} + + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = PySimpleRNN2(self.input_shape, self.output_shape) + + self.output = layers.mean(x=self.create_rnn_op(), **self.p_info) + + def create_rnn_op(self): + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot = layers.data( + shape=[self.input_dim], + data_type='float32', + name='h_boot', + **self.p_info) + h_boot.stop_gradient = False + + rnn = layers.StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre = rnn.memory(init=h_boot) + x_t = rnn.step_input(x) + + temp_l = layers.fc(input=x_t, + size=self.input_dim, + param_attr={'name': 'W'}, + bias_attr=False, + **self.p_info) + temp_r = layers.fc(input=h_pre, + size=self.input_dim, + param_attr={'name': 'U'}, + bias_attr=False, + **self.p_info) + + h = layers.sigmoid( + x=layers.elementwise_add( + x=temp_l, y=temp_r, **self.p_info), + **self.p_info) + + rnn.update_memory(h_pre, h) + rnn.output(h) + + return rnn() + + +class RecurrentOpMultipleMemoryTest(RecurrentOpTest1): + ''' + Test RNNOp with two memories + equation: + h_1 = h_pre_1 + h_2 = h_pre_2 + y = h_1 + h_2 + vars: + - x + memories: + - h_1, h_2 + outputs: + - y + ''' + + class PySimpleRNN3(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(RecurrentOpMultipleMemoryTest.PySimpleRNN3, self).__init__( + input_shape, output_shape) + + seq_len, batch_size, input_dim = input_shape + self.h_boot1 = np.random.normal(size=(batch_size, + input_dim)).astype("float32") + self.h_boot2 = np.random.normal(size=(batch_size, + input_dim)).astype("float32") + + men_dim = (seq_len, batch_size, input_dim) + self.mems1 = np.zeros(shape=men_dim).astype("float32") + self.mems2 = np.zeros(shape=men_dim).astype("float32") + + def step(self, step_id, x): + if step_id == 0: + pre_mem1 = self.h_boot1 + pre_mem2 = self.h_boot2 + else: + pre_mem1 = self.mems1[step_id - 1] + pre_mem2 = self.mems2[step_id - 1] + self.mems1[step_id] = pre_mem1 + self.mems2[step_id] = pre_mem2 + self.y[step_id] = self.mems1[step_id] + self.mems2[step_id] + x + + input_dim = 1 + batch_size = 1 + sent_len = 2 + + def setUp(self): + self.setup_program() + + self.data_field = {"x", "h_boot1", "h_boot2"} + + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = RecurrentOpMultipleMemoryTest.PySimpleRNN3( + self.input_shape, self.output_shape) + + self.output = layers.mean(x=self.create_rnn_op(), **self.p_info) + + def create_rnn_op(self): + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + h_boot1 = layers.data( + shape=[self.batch_size, self.input_dim], + data_type='float32', + name='h_boot1', + append_batch_size=False, + **self.p_info) + h_boot1.stop_gradient = False + h_boot2 = layers.data( + shape=[self.batch_size, self.input_dim], + data_type='float32', + name='h_boot2', + append_batch_size=False, + **self.p_info) + h_boot2.stop_gradient = False + + rnn = layers.StaticRNN(main_program=self.main_program) + with rnn.step(): + h_pre1 = rnn.memory(init=h_boot1) + h_pre2 = rnn.memory(init=h_boot2) + x_t = rnn.step_input(x) + + mem1 = layers.scale(x=h_pre1, scale=1.0, **self.p_info) + mem2 = layers.scale(x=h_pre2, scale=1.0, **self.p_info) + out = layers.sums(input=[mem1, x_t, mem2], **self.p_info) + + rnn.update_memory(h_pre1, mem1) + rnn.update_memory(h_pre2, mem2) + rnn.output(out) + + return rnn() + + +class RecurrentOpNoMemBootTest(RecurrentOpTest1): + ''' + Test RNNOp with two memories + equation: + mem = x + mem_pre + y = mem + vars: + - x + memories: + - mem + outputs: + - y + ''' + + class PySimpleRNN4(PyRNNBase): + def __init__(self, input_shape, output_shape): + super(RecurrentOpNoMemBootTest.PySimpleRNN4, self).__init__( + input_shape, output_shape) + men_dim = input_shape + self.mems = np.zeros(shape=men_dim).astype("float32") + + def step(self, step_id, x): + if step_id == 0: + pre_mem = np.zeros_like(x) + else: + pre_mem = self.mems[step_id - 1] + self.mems[step_id] = pre_mem + x + self.y[step_id] = self.mems[step_id] + + input_dim = 1 + batch_size = 1 + sent_len = 2 + + def setUp(self): + self.setup_program() + + self.data_field = {"x"} + + self.input_shape = (self.sent_len, self.batch_size, self.input_dim) + self.output_shape = (self.sent_len, self.batch_size, self.input_dim) + self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4(self.input_shape, + self.output_shape) + self.output = layers.mean(x=self.create_rnn_op(), **self.p_info) + print self.main_program + + def create_rnn_op(self): + x = layers.data( + shape=[self.sent_len, self.batch_size, self.input_dim], + data_type='float32', + name='x', + append_batch_size=False, + **self.p_info) + x.stop_gradient = False + + rnn = layers.StaticRNN(main_program=self.main_program) + with rnn.step(): + mem_pre = rnn.memory(shape=[-1, self.input_dim], batch_ref=x) + x_t = rnn.step_input(x) + mem = layers.elementwise_add(x=mem_pre, y=x_t, **self.p_info) + rnn.update_memory(mem_pre, mem) + rnn.output(mem) -class RecurrentGradientOpTest(unittest.TestCase): - def create_forward_op(self): - self.forward_op = RecurrentOp( - # inputs - inputs=["x"], - initial_states=["h_boot"], - step_net="stepnet", - # outputs - outputs=["h"], - step_scopes="step_scopes", - # attributes - ex_states=["h@pre"], - states=["h@alias"]) - - # create a stepnet for RNN - stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") - h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@alias") - - for op in [x_fc_op, h_fc_op, sum_op, sig_op]: - stepnet.append_op(op) - stepnet.complete_add_op(True) - self.forward_op.set_stepnet(stepnet) - - def create_gradient_op(self): - a = set() - backward_op = core.RecurrentOp.backward(self.forward_op, a) - - def test_grad(self): - self.create_forward_op() - self.create_gradient_op() + return rnn() if __name__ == '__main__': - exit( - 0 - ) # FIXME(qijun): https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957 unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rnn_helpers.py b/python/paddle/v2/framework/tests/test_rnn_helpers.py deleted file mode 100644 index be0ecfb129aa181229bc42d8d6818ad860991965..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_rnn_helpers.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest -from paddle.v2.framework.layers import * -from paddle.v2.framework.framework import g_program - - -class TestRNN(unittest.TestCase): - def test_rnn(self): - img = data( - shape=[ - 80, # sequence length - 22, # image height - 22 - ], # image width - data_type='float32', - name='image') - hidden = fc(input=img, size=100, act='sigmoid', num_flatten_dims=2) - self.assertEqual((-1, 80, 100), hidden.shape) - hidden = fc(input=hidden, size=100, act='sigmoid', num_flatten_dims=2) - self.assertEqual((-1, 80, 100), hidden.shape) - - rnn = StaticRNN() - with rnn.step(): - hidden = rnn.step_input(hidden) - self.assertEqual((-1, 100), hidden.shape) - memory = rnn.memory(shape=(-1, 32), dtype='float32', init_value=0.0) - - rnn_out = fc(input=[hidden, memory], size=32, act='sigmoid') - self.assertEqual((-1, 32), rnn_out.shape) - rnn.update_memory(memory, rnn_out) - rnn.output(rnn_out) - - out = rnn() - self.assertEqual((-1, 80, 32), out.shape) - print g_program - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py b/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py new file mode 100644 index 0000000000000000000000000000000000000000..731beff17cc96d26c2d9390a956c774b8676b179 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_rnn_memory_helper_op.py @@ -0,0 +1,130 @@ +import unittest + +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops +import numpy as np +import paddle.v2.framework.core as core + + +def create_tensor(np_data, place): + tensor = core.LoDTensor() + tensor.set(np_data, place) + return tensor + + +class RNNMemoryHelperOpTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.place = core.CPUPlace() + + self.X = self.program.global_block().create_var( + name='X', shape=[2, 3], dtype='float32') + self.Out = self.program.global_block().create_var( + name='Out', shape=[2, 3], dtype='float32') + self.program.global_block().append_op( + type='rnn_memory_helper', + inputs={"X": self.X}, + outputs={"Out": self.Out}, + attrs={}) + + def test_forward(self): + x_np = np.random.normal(size=(2, 3)).astype("float32") + self.feed_map = {'X': create_tensor(x_np, self.place)} + self.fetch_list = [self.Out] + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose(np.array(out[0]), x_np, rtol=1e-5) + + +class RNNMemoryHelperGradOpTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.place = core.CPUPlace() + + self.input_names = ['X', 'Out', 'Out@GRAD'] + self.input_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.input_names + } + + self.output_names = ['X@GRAD'] + self.output_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.output_names + } + + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}) + + def test_backward(self): + self.feed_map = { + name: create_tensor( + np.random.normal(size=(2, 3)).astype("float32"), self.place) + for name in self.input_names + } + self.fetch_list = [self.output_vars['X@GRAD']] + + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose(np.array(out[0]), self.feed_map['Out@GRAD'], rtol=1e-5) + + +class RNNMemoryHelperGradOpWithoutInputTest(unittest.TestCase): + def setUp(self): + self.program = Program() + self.fake_program = Program() + self.place = core.CPUPlace() + + self.input_names = ['X', 'Out'] + self.input_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.input_names + } + self.input_vars["Out@GRAD"] = \ + self.fake_program.global_block().create_var( + name="Out@GRAD", shape=[2, 3], dtype='float32') + + self.output_names = ['X@GRAD'] + self.output_vars = { + name: self.program.global_block().create_var( + name=name, shape=[2, 3], dtype='float32') + for name in self.output_names + } + + self.program.global_block().append_op( + type='rnn_memory_helper_grad', + inputs=self.input_vars, + outputs=self.output_vars, + attrs={}) + + def test_backward(self): + self.feed_map = { + name: create_tensor( + np.random.normal(size=(2, 3)).astype("float32"), self.place) + for name in ['X', 'Out'] + } + self.fetch_list = [self.output_vars['X@GRAD']] + + exe = Executor(self.place) + out = exe.run(self.program, + feed=self.feed_map, + fetch_list=self.fetch_list) + np.isclose( + np.array(out[0]), + np.zeros(shape=(2, 3)).astype("float32"), + rtol=1e-5) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_seq_concat_op.py b/python/paddle/v2/framework/tests/test_seq_concat_op.py index abd2ebf0b21a953b76155eb04c57a7b65ac53cbc..dccc6ed8afe2315da74f6886878b15d58b26b3c9 100644 --- a/python/paddle/v2/framework/tests/test_seq_concat_op.py +++ b/python/paddle/v2/framework/tests/test_seq_concat_op.py @@ -2,9 +2,36 @@ import unittest import numpy as np import sys from op_test import OpTest +exit(0) -class TestConcatOp(OpTest): +def to_abs_lod(lod): + if len(lod) == 0 or len(lod) == 1: + return lod + import copy + new_lod = copy.deepcopy(lod) + for idx, val in enumerate(lod[0]): + new_lod[0][idx] = lod[1][val] + return new_lod + + +def seq_concat(inputs, level): + lod0 = inputs['X'][0][1][1] + lod1 = inputs['X'][1][1][1] + x0 = inputs['X'][0][1][0] + x1 = inputs['X'][1][1][0] + level_idx = len(lod0) - level - 1 + outs = [] + for i in range(len(lod0[level_idx]) - 1): + sub_x0 = x0[to_abs_lod(lod0)[level_idx][i]:to_abs_lod(lod0)[level_idx][ + i + 1], :] + sub_x1 = x1[to_abs_lod(lod1)[level_idx][i]:to_abs_lod(lod1)[level_idx][ + i + 1], :] + outs.append(np.concatenate((sub_x0, sub_x1), axis=0)) + return np.concatenate(outs, axis=0) + + +class TestSeqConcatOp(OpTest): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') @@ -15,13 +42,7 @@ class TestConcatOp(OpTest): level = 1 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(4): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) - - self.outputs = {'Out': np.concatenate(outs, axis=0)} + self.outputs = {'Out': (np.concatenate([x0, x1], axis=1), lod0)} def setUp(self): self.op_type = "sequence_concat" @@ -34,46 +55,50 @@ class TestConcatOp(OpTest): self.check_grad(['x0'], 'Out') -class TestConcatOpDiffLod(TestConcatOp): +class TestSeqConcatOpLevelZeroNestedSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 6, 3)).astype('float32') lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] - x1 = np.random.random((5, 6, 3)).astype('float32') - lod1 = [[0, 3, 5], [0, 1, 2, 3, 5]] + x1 = np.random.random((7, 6, 3)).astype('float32') + lod1 = [[0, 2, 4], [0, 1, 3, 5, 7]] axis = 0 - level = 1 + level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(4): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) + out_lod = [[0, 2, 4], [0, 2, 5, 8, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} - self.outputs = {'Out': np.concatenate(outs, axis=0)} + +class TestSeqConcatOplevelOneNestedSequence(TestSeqConcatOp): + def set_data(self): + # two level, batch size is 3 + x0 = np.random.random((4, 6, 3)).astype('float32') + lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] + x1 = np.random.random((7, 6, 3)).astype('float32') + lod1 = [[0, 3, 4], [0, 1, 3, 5, 7]] + axis = 0 + level = 1 + self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} + self.attrs = {'axis': axis, 'level': level} + out_lod = [[0, 5, 8], [0, 1, 2, 3, 5, 7, 8, 9, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} -class TestConcatOpLevelZero(TestConcatOp): +class TestSeqConcatOpLevelZeroSequence(TestSeqConcatOp): def set_data(self): # two level, batch size is 3 x0 = np.random.random((4, 3, 4)).astype('float32') - lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]] - x1 = np.random.random((5, 3, 4)).astype('float32') - lod1 = [[0, 3, 5], [0, 1, 3, 4, 5]] + lod0 = [[0, 1, 2, 3, 4]] + x1 = np.random.random((7, 3, 4)).astype('float32') + lod1 = [[0, 1, 3, 5, 7]] axis = 0 level = 0 self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]} self.attrs = {'axis': axis, 'level': level} - outs = [] - for i in range(2): - sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :] - sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :] - outs.append(np.concatenate((sub_x0, sub_x1), axis=axis)) - - self.outputs = {'Out': np.concatenate(outs, axis=0)} + out_lod = [[0, 2, 5, 8, 11]] + self.outputs = {'Out': (seq_concat(self.inputs, level), out_lod)} if __name__ == '__main__': - sys.exit(0) unittest.main() diff --git a/python/paddle/v2/framework/tests/test_seq_pool.py b/python/paddle/v2/framework/tests/test_seq_pool.py index efc4920124afb539017a3b3f211c7320da68ffef..512d8b315f29cecf79ae274dca491c240f3447a1 100644 --- a/python/paddle/v2/framework/tests/test_seq_pool.py +++ b/python/paddle/v2/framework/tests/test_seq_pool.py @@ -29,6 +29,9 @@ class TestSeqAvgPool(OpTest): self.check_output() def test_check_grad(self): + # Remove MaxIndex after check_grad is refined. + self.outputs['MaxIndex'] = \ + np.zeros(self.outputs['Out'].shape).astype('int32') self.check_grad(["X"], "Out") @@ -85,31 +88,53 @@ class TestSeqSqrtPool2D(TestSeqAvgPool2D): out[i] = np.reshape(sub_x.sum(axis=0) / np.sqrt(len), (3, 17)) def test_check_grad(self): + # Remove MaxIndex after check_grad is refined. + self.outputs['MaxIndex'] = \ + np.zeros(self.outputs['Out'].shape).astype('int32') self.check_grad(["X"], "Out", max_relative_error=0.06) class TestSeqMaxPool(TestSeqAvgPool): + def set_data(self): + self.op_type = 'sequence_pool' + x = np.random.uniform(0.1, 1, [13, 23]).astype('float32') + lod = [[0, 4, 5, 8, 13]] + for i in range(4): + l = lod[0][i + 1] - lod[0][i] + x[lod[0][i] + np.random.randint(l), :] += 2.0 + + self.inputs = {'X': (x, lod)} + + out = np.zeros((4, 23)).astype('float32') + self.outputs = {'Out': out} + return x, lod, out + def compute(self, x, lod, out): self.attrs = {'pooltype': "MAX"} for i in range(4): sub_x = x[lod[0][i]:lod[0][i + 1], :] out[i] = np.amax(sub_x, axis=0) - def test_check_grad(self): - # Remove MaxPool2D from gradient check to confirm the success of CI. - return - class TestSeqMaxPool2D(TestSeqAvgPool2D): + def set_data(self): + self.op_type = 'sequence_pool' + x = np.random.uniform(0.1, 1, [13, 3, 11]).astype('float32') + lod = [[0, 4, 5, 8, 13]] + self.inputs = {'X': (x, lod)} + for i in range(4): + l = lod[0][i + 1] - lod[0][i] + x[lod[0][i] + np.random.randint(l), :] += 1.0 + + out = np.zeros((4, 3, 11)).astype('float32') + self.outputs = {'Out': out} + return x, lod, out + def compute(self, x, lod, out): self.attrs = {'pooltype': "MAX"} for i in range(4): - sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 17)) - out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 17)) - - def test_check_grad(self): - # Remove MaxPool2D from gradient check to confirm the success of CI. - return + sub_x = np.reshape(x[lod[0][i]:lod[0][i + 1], :], (-1, 3 * 11)) + out[i] = np.reshape(np.amax(sub_x, axis=0), (3, 11)) class TestSeqLastPool(TestSeqAvgPool): diff --git a/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py b/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py new file mode 100644 index 0000000000000000000000000000000000000000..2090455b969806685b525f1e588b6570e3072430 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_shrink_rnn_memory.py @@ -0,0 +1,47 @@ +import unittest +import paddle.v2.framework.core as core +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.layers as layers +from paddle.v2.framework.backward import append_backward_ops +from paddle.v2.framework.framework import g_main_program +import numpy + + +class TestShrinkRNNMemory(unittest.TestCase): + def test_shrink_rnn_memory(self): + x = layers.data('x', shape=[100], data_type='float32') + x.stop_gradient = False + table = layers.lod_rank_table(x=x) + i = layers.zeros(dtype='int64', shape=[1]) + mem1 = layers.shrink_memory(x=x, i=i, table=table) + i = layers.increment(x=i) + i.stop_gradient = True + mem2 = layers.shrink_memory(x=mem1, i=i, table=table) + i = layers.increment(x=i) + i.stop_gradient = True + mem3 = layers.shrink_memory(x=mem2, i=i, table=table) + + cpu = core.CPUPlace() + tensor = core.LoDTensor() + tensor.set_lod([[0, 2, 5, 6]]) + tensor_np = numpy.random.random(size=(3, 100)).astype('float32') + tensor.set(tensor_np, cpu) + exe = Executor(cpu) + outs = map(numpy.array, + exe.run(feed={'x': tensor}, fetch_list=[mem1, mem2, mem3])) + self.assertTrue(numpy.allclose(tensor_np[0:3], outs[0])) + self.assertTrue(numpy.allclose(tensor_np[0:2], outs[1])) + self.assertTrue(numpy.allclose(tensor_np[0:1], outs[2])) + + mem3_mean = layers.mean(x=mem3) + append_backward_ops(loss=mem3_mean) + x_grad = map(numpy.array, + exe.run(feed={'x': tensor}, + fetch_list=[ + g_main_program.global_block().var('x@GRAD') + ]))[0] + self.assertAlmostEqual(1.0, x_grad.sum(), delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_softmax_with_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_softmax_with_cross_entropy_op.py index f93feb20696f126423bc9412eab3b4aa41b19426..c2f07f9096c69f3d4977f9444bdd5dcda8028973 100644 --- a/python/paddle/v2/framework/tests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/v2/framework/tests/test_softmax_with_cross_entropy_op.py @@ -12,30 +12,30 @@ class TestSoftmaxWithCrossEntropyOp(OpTest): def setUp(self): self.op_type = "softmax_with_cross_entropy" - batch_size = 3 + batch_size = 2 class_num = 37 logits = np.random.uniform(0.1, 1.0, - [batch_size, class_num]).astype("float32") + [batch_size, class_num]).astype("float64") softmax = np.apply_along_axis(stable_softmax, 1, logits) - labels = np.random.randint(0, class_num, [batch_size, 1], dtype="int32") + labels = np.random.randint(0, class_num, [batch_size, 1], dtype="int64") cross_entropy = np.asmatrix( [[-np.log(softmax[i][labels[i][0]])] for i in range(softmax.shape[0])], - dtype="float32") + dtype="float64") self.inputs = {"Logits": logits, "Label": labels} self.outputs = { - "Softmax": softmax.astype('float32'), - "Loss": cross_entropy.astype('float32') + "Softmax": softmax.astype("float64"), + "Loss": cross_entropy.astype("float64") } def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(["Logits"], "Loss", max_relative_error=0.05) + self.check_grad(["Logits"], "Loss") class TestSoftmaxWithCrossEntropyOp2(OpTest): @@ -49,19 +49,19 @@ class TestSoftmaxWithCrossEntropyOp2(OpTest): class_num = 37 logits = np.random.uniform(0.1, 1.0, - [batch_size, class_num]).astype("float32") + [batch_size, class_num]).astype("float64") softmax = np.apply_along_axis(stable_softmax, 1, logits) labels = np.random.uniform(0.1, 1.0, - [batch_size, class_num]).astype("float32") + [batch_size, class_num]).astype("float64") labels /= np.sum(labels, axis=1, keepdims=True) cross_entropy = (-labels * np.log(softmax)).sum( - axis=1, keepdims=True).astype("float32") + axis=1, keepdims=True).astype("float64") self.inputs = {"Logits": logits, "Label": labels} self.outputs = { - "Softmax": softmax.astype('float32'), - "Loss": cross_entropy.astype('float32') + "Softmax": softmax.astype("float64"), + "Loss": cross_entropy.astype("float64") } self.attrs = {"soft_label": True} @@ -69,9 +69,8 @@ class TestSoftmaxWithCrossEntropyOp2(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(["Logits"], "Loss", max_relative_error=0.05) + self.check_grad(["Logits"], "Loss") if __name__ == "__main__": - exit(0) # FIXME: xe has bug unittest.main() diff --git a/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py b/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py new file mode 100644 index 0000000000000000000000000000000000000000..6ba1e568249d4a72820cc26193a8e0e030ae5f7c --- /dev/null +++ b/python/paddle/v2/framework/tests/test_split_and_merge_lod_tensor_op.py @@ -0,0 +1,181 @@ +import unittest +import paddle.v2.framework.core as core +import numpy as np +import paddle.v2.framework.layers as layers +from paddle.v2.framework.framework import Program +from paddle.v2.framework.executor import Executor +from paddle.v2.framework.backward import append_backward_ops + + +class TestCPULoDTensorArrayOps(unittest.TestCase): + def place(self): + return core.CPUPlace() + + def test_split_and_merge_lod_tensor_no_lod(self): + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place()) + + mask_np = np.array([0, 0, 1, 1, 1, 1, 0, 0, 0, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, self.place()) + + expect_true_tensor = np.array([2, 3, 4, 5]).astype('int32') + expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1) + expect_true = core.LoDTensor() + expect_true.set(expect_true_tensor, self.place()) + + expect_false_tensor = np.array([0, 1, 6, 7, 8, 9]).astype('int32') + expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1) + + expect_false = core.LoDTensor() + expect_false.set(expect_false_tensor, self.place()) + + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor) + + def test_split_and_merge_lod_tensor_level_0(self): + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('int32'), self.place()) + tensor.set_lod([[0, 3, 9, 10]]) + + mask_np = np.array([0, 1, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, self.place()) + + expect_true_tensor = np.array([3, 4, 5, 6, 7, 8]).astype('int32') + expect_true_tensor = np.expand_dims(expect_true_tensor, axis=1) + expect_true = core.LoDTensor() + expect_true.set(expect_true_tensor, self.place()) + expect_true.set_lod([[0, 6]]) + + expect_false_tensor = np.array([0, 1, 2, 9]).astype('int32') + expect_false_tensor = np.expand_dims(expect_false_tensor, axis=1) + expect_false_lod = [[0, 3, 4]] + + expect_false = core.LoDTensor() + expect_false.set(expect_false_tensor, self.place()) + expect_false.set_lod(expect_false_lod) + + self.main( + tensor=tensor, + mask=mask, + expect_true=expect_true, + expect_false=expect_false, + expect_out=tensor) + + def main(self, tensor, mask, expect_true, expect_false, expect_out, + level=0): + place = self.place() + program = Program() + x = layers.data(name='x', shape=[1], main_program=program) + x.persistable = True + + y = layers.data(name='y', shape=[1], main_program=program) + y.persistable = True + + out_true, out_false = layers.split_lod_tensor( + input=x, mask=y, level=level, main_program=program) + out_true.persistable = True + out_false.persistable = True + + out = layers.merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + main_program=program) + + out.persistable = True + + exe = Executor(place) + scope = core.Scope() + exe.run(program, feed={'x': tensor, 'y': mask}, scope=scope) + + var_true = scope.find_var(out_true.name).get_tensor() + + var_false = scope.find_var(out_false.name).get_tensor() + + var_out = scope.find_var(out.name).get_tensor() + + self.check_tensor_same(var_true, expect_true) + self.check_tensor_same(var_false, expect_false) + self.check_tensor_same(var_out, expect_out) + + def check_tensor_same(self, actual, expect): + self.assertTrue(np.allclose(np.array(actual), np.array(expect))) + self.assertEqual(actual.lod(), expect.lod()) + + +class TestCPUSplitMergeLoDTensorGrad(unittest.TestCase): + def test_grad(self): + place = core.CPUPlace() + program = Program() + + x = layers.data( + name='x', + shape=[1], + data_type='float32', + main_program=program, + stop_gradient=False) + y = layers.data( + name='y', + shape=[1], + data_type='bool', + main_program=program, + stop_gradient=False) + + level = 0 + + out_true, out_false = layers.split_lod_tensor( + input=x, mask=y, level=level, main_program=program) + out = layers.merge_lod_tensor( + in_true=out_true, + in_false=out_false, + mask=y, + x=x, + level=level, + main_program=program) + mean = layers.mean(x=out, main_program=program) + + append_backward_ops(mean) + + tensor = core.LoDTensor() + tensor.set(np.arange(10).reshape(10, 1).astype('float32'), place) + tensor.set_lod([[0, 3, 9, 10]]) + + mask_np = np.array([0, 1, 0]).astype('bool') + mask_np = np.expand_dims(mask_np, axis=1) + + mask = core.LoDTensor() + mask.set(mask_np, place) + + exe = Executor(place) + scope = core.Scope() + + g_vars = program.global_block().var(x.name + "@GRAD") + g_out = [ + item.sum() + for item in map(np.array, + exe.run(program, + feed={'x': tensor, + 'y': mask}, + fetch_list=[g_vars], + scope=scope)) + ] + + g_out_sum = np.array(g_out).sum() + + self.assertAlmostEqual(1.0, g_out_sum, delta=0.1) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_variable.py b/python/paddle/v2/framework/tests/test_variable.py index c670ca19afbd778747303cb002666aa2a5e62c37..03115f10a5a494424c6f8310c544c569be818e5b 100644 --- a/python/paddle/v2/framework/tests/test_variable.py +++ b/python/paddle/v2/framework/tests/test_variable.py @@ -1,5 +1,5 @@ import unittest -from paddle.v2.framework.framework import Variable, g_program, Program +from paddle.v2.framework.framework import Variable, g_main_program, Program import paddle.v2.framework.core as core import numpy as np @@ -18,7 +18,7 @@ class TestVariable(unittest.TestCase): self.assertRaises(ValueError, lambda: convert("int8")) def test_var(self): - b = g_program.current_block() + b = g_main_program.current_block() w = b.create_var( dtype="float64", shape=[784, 100], lod_level=0, name="fc.w") self.assertNotEqual(str(w), "") diff --git a/python/paddle/v2/framework/tests/test_while_op.py b/python/paddle/v2/framework/tests/test_while_op.py new file mode 100644 index 0000000000000000000000000000000000000000..1c344eae49705ecce586154c30c4d4f770022e7e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_while_op.py @@ -0,0 +1,68 @@ +import unittest +import paddle.v2.framework.layers as layers +from paddle.v2.framework.executor import Executor +import paddle.v2.framework.core as core +import numpy + + +class TestWhileOp(unittest.TestCase): + def test_simple_forward(self): + d0 = layers.data( + "d0", shape=[10], append_batch_size=False, data_type='float32') + d1 = layers.data( + "d1", shape=[10], append_batch_size=False, data_type='float32') + d2 = layers.data( + "d2", shape=[10], append_batch_size=False, data_type='float32') + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + init = layers.zeros(shape=[10], dtype='float32') + mem_array = layers.array_write(init, i=i) + data_array = layers.array_write(x=d0, i=i) + + i = layers.increment(i) + layers.array_write(d1, i, array=data_array) + + i = layers.increment(i) + layers.array_write(d2, i, array=data_array) + + i = layers.zeros(shape=[1], dtype='int64') + i.stop_gradient = True + + array_len = layers.fill_constant(shape=[1], dtype='int64', value=3) + cond = layers.less_than(x=i, y=array_len) + + while_op = layers.While(cond=cond) + with while_op.block(): + d = layers.array_read(array=data_array, i=i) + prev = layers.array_read(array=mem_array, i=i) + i = layers.increment(x=i, in_place=True) + result = layers.sums(input=[d, prev]) + layers.array_write(result, i=i, array=mem_array) + layers.less_than(x=i, y=array_len, cond=cond) + sum_result = layers.array_read(mem_array, i=array_len) + + cpu = core.CPUPlace() + exe = Executor(cpu) + d = [] + + for i in xrange(3): + d.append(numpy.random.random(size=[10]).astype('float32')) + + d_tensor = [] + for item in d: + t = core.LoDTensor() + t.set(item, cpu) + d_tensor.append(t) + + outs = map(numpy.array, + exe.run(feed={ + 'd0': d_tensor[0], + 'd1': d_tensor[1], + 'd2': d_tensor[2] + }, + fetch_list=[sum_result])) + self.assertAlmostEqual(numpy.sum(d), numpy.sum(outs[0]), delta=0.01) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/image.py b/python/paddle/v2/image.py index 965d965335a56a97448bd8c738b03eceaee550e2..7408ea8ef611ddfa74dc5bb6ef45d4e0ccb9d141 100644 --- a/python/paddle/v2/image.py +++ b/python/paddle/v2/image.py @@ -1,33 +1,35 @@ -import numpy as np -try: - import cv2 -except ImportError: - cv2 = None -import os -import tarfile -import cPickle - -__all__ = [ - "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", - "random_crop", "left_right_flip", "simple_transform", "load_and_transform", - "batch_images_from_tar" -] """ This file contains some common interfaces for image preprocess. Many users are confused about the image layout. We introduce the image layout as follows. - CHW Layout + - The abbreviations: C=channel, H=Height, W=Width - The default layout of image opened by cv2 or PIL is HWC. PaddlePaddle only supports the CHW layout. And CHW is simply a transpose of HWC. It must transpose the input image. - Color format: RGB or BGR + OpenCV use BGR color format. PIL use RGB color format. Both formats can be used for training. Noted that, the format should be keep consistent between the training and inference peroid. """ +import numpy as np +try: + import cv2 +except ImportError: + cv2 = None +import os +import tarfile +import cPickle + +__all__ = [ + "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", + "random_crop", "left_right_flip", "simple_transform", "load_and_transform", + "batch_images_from_tar" +] def batch_images_from_tar(data_file, @@ -36,17 +38,18 @@ def batch_images_from_tar(data_file, num_per_batch=1024): """ Read images from tar file and batch them into batch file. - param data_file: path of image tar file - type data_file: string - param dataset_name: 'train','test' or 'valid' - type dataset_name: string - param img2label: a dic with image file name as key + + :param data_file: path of image tar file + :type data_file: string + :param dataset_name: 'train','test' or 'valid' + :type dataset_name: string + :param img2label: a dic with image file name as key and image's label as value - type img2label: dic - param num_per_batch: image number per batch file - type num_per_batch: int - return: path of list file containing paths of batch file - rtype: string + :type img2label: dic + :param num_per_batch: image number per batch file + :type num_per_batch: int + :return: path of list file containing paths of batch file + :rtype: string """ batch_dir = data_file + "_batch" out_path = "%s/%s" % (batch_dir, dataset_name) @@ -99,14 +102,16 @@ def load_image_bytes(bytes, is_color=True): Example usage: .. code-block:: python + with open('cat.jpg') as f: im = load_image_bytes(f.read()) :param bytes: the input image bytes array. - :type file: str + :type bytes: str :param is_color: If set is_color True, it will load and return a color image. Otherwise, it will load and return a gray image. + :type is_color: bool """ flag = 1 if is_color else 0 file_bytes = np.asarray(bytearray(bytes), dtype=np.uint8) @@ -121,6 +126,7 @@ def load_image(file, is_color=True): Example usage: .. code-block:: python + im = load_image('cat.jpg') :param file: the input image path. @@ -128,6 +134,7 @@ def load_image(file, is_color=True): :param is_color: If set is_color True, it will load and return a color image. Otherwise, it will load and return a gray image. + :type is_color: bool """ # cv2.IMAGE_COLOR for OpenCV3 # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version @@ -147,6 +154,7 @@ def resize_short(im, size): Example usage: .. code-block:: python + im = load_image('cat.jpg') im = resize_short(im, 256) @@ -175,6 +183,7 @@ def to_chw(im, order=(2, 0, 1)): Example usage: .. code-block:: python + im = load_image('cat.jpg') im = resize_short(im, 256) im = to_chw(im) @@ -196,6 +205,7 @@ def center_crop(im, size, is_color=True): Example usage: .. code-block:: python + im = center_crop(im, 224) :param im: the input image with HWC layout. @@ -223,6 +233,7 @@ def random_crop(im, size, is_color=True): Example usage: .. code-block:: python + im = random_crop(im, 224) :param im: the input image with HWC layout. @@ -251,6 +262,7 @@ def left_right_flip(im): Example usage: .. code-block:: python + im = left_right_flip(im) :paam im: input image with HWC layout @@ -275,6 +287,7 @@ def simple_transform(im, Example usage: .. code-block:: python + im = simple_transform(im, 256, 224, True) :param im: The input image with HWC layout. @@ -285,6 +298,11 @@ def simple_transform(im, :type crop_size: int :param is_train: Whether it is training or not. :type is_train: bool + :param is_color: whether the image is color or not. + :type is_color: bool + :param mean: the mean values, which can be element-wise mean values or + mean values per channel. + :type mean: numpy array | list """ im = resize_short(im, resize_size) if is_train: @@ -324,6 +342,7 @@ def load_and_transform(filename, Example usage: .. code-block:: python + im = load_and_transform('cat.jpg', 256, 224, True) :param filename: The file name of input image. @@ -334,6 +353,11 @@ def load_and_transform(filename, :type crop_size: int :param is_train: Whether it is training or not. :type is_train: bool + :param is_color: whether the image is color or not. + :type is_color: bool + :param mean: the mean values, which can be element-wise mean values or + mean values per channel. + :type mean: numpy array | list """ im = load_image(filename) im = simple_transform(im, resize_size, crop_size, is_train, is_color, mean) diff --git a/python/paddle/v2/optimizer.py b/python/paddle/v2/optimizer.py index 29f0945eb4c88eab8fa9ee83f455190dfd473aa4..caef5f484e2d629f2298ced457e89ff93a536311 100644 --- a/python/paddle/v2/optimizer.py +++ b/python/paddle/v2/optimizer.py @@ -11,11 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Optimizers(update equation) for SGD method. - -TODO(yuyang18): Complete comments. -""" import paddle.trainer_config_helpers.config_parser_utils as config_parser_utils import paddle.trainer_config_helpers.optimizers as v1_optimizers @@ -101,32 +96,37 @@ class Optimizer(object): class Momentum(Optimizer): """ - SGD Optimizer. - - SGD is an optimization method, trying to find a neural network that - minimize the "cost/error" of it by iteration. In paddle's implementation - SGD Optimizer is synchronized, which means all gradients will be wait to - calculate and reduced into one gradient, then do optimize operation. + Momentum Optimizer. - The neural network consider the learning problem of minimizing an objective - function, that has the form of a sum + When sparse=False, the momentum update formula is as follows: .. math:: - Q(w) = \\sum_{i}^{n} Q_i(w) + v_{t} &= k * v_{t-1} - \\gamma_t (g_{t} + \\lambda w_{t-1}) \\\\ + w_{t} &= w_{t-1} + v_{t} \\\\ - The value of function Q sometimes is the cost of neural network (Mean - Square Error between prediction and label for example). The function Q is - parametrised by w, the weight/bias of neural network. And weights is what to - be learned. The i is the i-th observation in (trainning) data. + where, :math:`k` is momentum, :math:`\\lambda` is decay rate, + :math:`\\gamma_t` is learning rate at the t'th iteration. + :math:`w_{t}` is the weight as the t'th iteration. + And the :math:`v_{t}` is the history momentum variable. - So, the SGD method will optimize the weight by + When sparse=True, the update scheme: .. math:: - w = w - \\eta \\nabla Q(w) = w - \\eta \\sum_{i}^{n} \\nabla Q_i(w) - - where :math:`\\eta` is learning rate. And :math:`n` is batch size. + \\alpha_t &= \\alpha_{t-1} / k \\\\ + \\beta_t &= \\beta_{t-1} / (1 + \\lambda \\gamma_t) \\\\ + u_t &= u_{t-1} - \\alpha_t \\gamma_t g_t \\\\ + v_t &= v_{t-1} + \\tau_{t-1} \\alpha_t \\gamma_t g_t \\\\ + \\tau_t &= \\tau_{t-1} + \\beta_t / \\alpha_t + + where :math:`k` is momentum, :math:`\\lambda` is decay rate, + :math:`\\gamma_t` is learning rate at the t'th iteration. + + :param momentum: the momentum factor. + :type momentum: float + :param sparse: with sparse support or not, False by default. + :type sparse: bool """ def __init__(self, momentum=None, sparse=False, **kwargs): @@ -146,7 +146,7 @@ class Adam(Optimizer): m(w, t) & = \\beta_1 m(w, t-1) + (1 - \\beta_1) \\nabla Q_i(w) \\\\ v(w, t) & = \\beta_2 v(w, t-1) + (1 - \\beta_2)(\\nabla Q_i(w)) ^2 \\\\ - w & = w - \\frac{\\eta}{\\sqrt{v(w,t) + \\epsilon}} + w & = w - \\frac{\\eta m(w, t)}{\\sqrt{v(w,t) + \\epsilon}} :param beta1: the :math:`\\beta_1` in equation. :type beta1: float diff --git a/python/paddle/v2/plot/plot.py b/python/paddle/v2/plot/plot.py index 6f7bd039b07db4832295c2374293bffa588eb4ef..c18e63dd5f60481ba804738a6a9238dfea35d9f3 100644 --- a/python/paddle/v2/plot/plot.py +++ b/python/paddle/v2/plot/plot.py @@ -56,7 +56,7 @@ class Ploter(object): assert isinstance(data, PlotData) data.append(step, value) - def plot(self): + def plot(self, path=None): if self.__plot_is_disabled__(): return @@ -68,8 +68,11 @@ class Ploter(object): titles.append(title) self.plt.plot(data.step, data.value) self.plt.legend(titles, loc='upper left') - self.display.clear_output(wait=True) - self.display.display(self.plt.gcf()) + if path is None: + self.display.clear_output(wait=True) + self.display.display(self.plt.gcf()) + else: + self.plt.savefig(path) self.plt.gcf().clear() def reset(self): diff --git a/python/requirements.txt b/python/requirements.txt index e19453c25da1ec78773c00a72b8e517b0d798fff..daf3f368b92408408897e33223118fe3647aa6de 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -7,3 +7,4 @@ rarfile scipy>=0.19.0 Pillow nltk>=3.2.2 +graphviz diff --git a/python/setup.py.in b/python/setup.py.in index 87b3823e52604b889cdee76bc696a1ae9b9de802..5348c2d8d7e9b5adc5fe93e2943bef149ba047cc 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,4 +1,4 @@ -from setuptools import setup, Distribution +from setuptools import setup, Distribution, Extension class BinaryDistribution(Distribution): def has_ext_modules(foo): return True @@ -41,6 +41,7 @@ setup(name='paddlepaddle', description='Parallel Distributed Deep Learning', install_requires=setup_requires, packages=packages, + ext_modules=[Extension('_foo', ['stub.cc'])], package_data={ 'paddle.v2.master': ['libpaddle_master.so'], 'paddle.v2.framework': ['core.so'], @@ -54,6 +55,5 @@ setup(name='paddlepaddle', 'py_paddle': '${PADDLE_SOURCE_DIR}/paddle/py_paddle' }, scripts=paddle_bins, - distclass=BinaryDistribution, data_files=[(paddle_rt_lib_dir, paddle_rt_libs)] )