diff --git a/.gitignore b/.gitignore index 4f21fefda9f64a0392881971a715b97c234030e3..351b8204100dfd71e94cb3efa2e946b44b9e4285 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ CMakeFiles cmake_install.cmake paddle/.timestamp python/paddlepaddle.egg-info/ +paddle/pybind/pybind.h diff --git a/.travis.yml b/.travis.yml index e217c8f5a740ef5ab7315656ed7839ffa219c805..d0e2696f100e55f320e410afd6a3038db647f76f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,10 +36,6 @@ before_install: # protobuf version. - sudo pip install -r $TRAVIS_BUILD_DIR/python/requirements.txt - sudo pip install wheel sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit LinkChecker - - curl https://glide.sh/get | bash - - eval "$(GIMME_GO_VERSION=1.8.3 gimme)" - - go get -u github.com/alecthomas/gometalinter - - gometalinter --install - | function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } script: diff --git a/CMakeLists.txt b/CMakeLists.txt index 5739c2a26039426ab544f762e9401445f01e7de7..4921226ec1c90a969fa1cfc383823820500c7757 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ if(NOT CMAKE_CROSSCOMPILING) endif(NOT CMAKE_CROSSCOMPILING) find_package(Git REQUIRED) find_package(Threads REQUIRED) -if(NOT ANDROID) +if(NOT ANDROID AND NOT IOS) find_package(Boost QUIET) endif() @@ -64,27 +64,29 @@ if(NOT CMAKE_BUILD_TYPE) FORCE) endif() -if(ANDROID) - if(${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") - message(FATAL_ERROR "Unsupport standalone toolchains with Android API level lower than 16") - elseif(${CMAKE_SYSTEM_VERSION} VERSION_LESS "21") - # TODO: support glog for Android api 16 ~ 19 in the future - message(WARNING "Using the unofficial git repository instead") +if(ANDROID OR IOS) + if(ANDROID) + if(${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") + message(FATAL_ERROR "Unsupport standalone toolchains with Android API level lower than 16") + elseif(${CMAKE_SYSTEM_VERSION} VERSION_LESS "21") + # TODO: support glog for Android api 16 ~ 19 in the future + message(WARNING "Using the unofficial git repository instead") + endif() endif() set(WITH_GPU OFF CACHE STRING - "Disable GPU when cross-compiling for Android" FORCE) + "Disable GPU when cross-compiling for Android and iOS" FORCE) set(WITH_AVX OFF CACHE STRING - "Disable AVX when cross-compiling for Android" FORCE) + "Disable AVX when cross-compiling for Android and iOS" FORCE) set(WITH_PYTHON OFF CACHE STRING - "Disable PYTHON when cross-compiling for Android" FORCE) + "Disable PYTHON when cross-compiling for Android and iOS" FORCE) set(WITH_RDMA OFF CACHE STRING - "Disable RDMA when cross-compiling for Android" FORCE) + "Disable RDMA when cross-compiling for Android and iOS" FORCE) set(WITH_MKLDNN OFF CACHE STRING - "Disable MKLDNN when cross-compiling for Android" FORCE) + "Disable MKLDNN when cross-compiling for Android and iOS" FORCE) set(WITH_MKLML OFF CACHE STRING - "Disable MKLML package when cross-compiling for Android" FORCE) -endif(ANDROID) + "Disable MKLML package when cross-compiling for Android and iOS" FORCE) +endif() set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING "A path setting third party libraries download & build directories.") diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 854066fd1d205c337fbdbe08997d88251095c799..8fdc382f0c1c453a01dba884a3dad216e1c3092c 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -171,3 +171,10 @@ if (REFERENCE_CBLAS_INCLUDE_DIR AND REFERENCE_CBLAS_LIBRARY) add_definitions(-DPADDLE_USE_REFERENCE_CBLAS) message(STATUS "Found reference-cblas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})") endif() + +if(IOS_USE_VECLIB_FOR_BLAS AND VECLIB_FOUND) + set(CBLAS_FOUND ON) + set(CBLAS_PROVIDER vecLib) + set(CBLAS_INC_DIR ${VECLIB_INC_DIR}) + add_definitions(-DPADDLE_USE_VECLIB) +endif() diff --git a/cmake/cross_compiling/ios.cmake b/cmake/cross_compiling/ios.cmake new file mode 100644 index 0000000000000000000000000000000000000000..0b38943952f7fb9052368fe95eb31dd7592d8a47 --- /dev/null +++ b/cmake/cross_compiling/ios.cmake @@ -0,0 +1,350 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a toolchain file for cross-compiling for iOS, and the +# configuration largely refers to public toolchain file: +# https://raw.githubusercontent.com/leetal/ios-cmake/master/ios.toolchain.cmake +# and +# https://github.com/cristeab/ios-cmake +# +# Supports options: +# IOS_PLATFORM = OS (default) or SIMULATOR +# This decides if SDKS will be selected from the iPhoneOS.platform or iPhoneSimulator.platform folders +# OS - the default, used to build for iPhone and iPad physical devices, which have an arm arch. +# SIMULATOR - used to build for the Simulator platforms, which have an x86 arch. +# IOS_ARCH +# The archectures wanted to support, such "arm64", "armv7;arm64" +# IOS_DEPLOYMENT_TARGET +# The minimum iOS deployment version, such as "7.0" +# IOS_ENABLE_BITCODE = ON (default) or OFF +# IOS_USE_VECLIB_FOR_BLAS = OFF (default) or ON +# IOS_DEVELOPER_ROOT = automatic(default) or /path/to/platform/Developer folder +# By default this location is automatcially chosen based on the IOS_PLATFORM value above. +# If set manually, it will override the default location and force the user of a particular Developer Platform +# IOS_SDK_ROOT = automatic(default) or /path/to/platform/Developer/SDKs/SDK folder +# By default this location is automatcially chosen based on the IOS_DEVELOPER_ROOT value. +# In this case it will always be the most up-to-date SDK found in the IOS_DEVELOPER_ROOT path. +# If set manually, this will force the use of a specific SDK version + +# Macros: +# set_xcode_property (TARGET XCODE_PROPERTY XCODE_VALUE) +# A convenience macro for setting xcode specific properties on targets +# example: set_xcode_property (myioslib IPHONEOS_DEPLOYMENT_TARGET "3.1") +# find_host_package (PROGRAM ARGS) +# A macro used to find executable programs on the host system, not within the iOS environment. +# Thanks to the android-cmake project for providing the command + +if(NOT IOS) + return() +endif() + +set(CMAKE_SYSTEM_NAME Darwin) + +# Get the Xcode version being used. +execute_process(COMMAND xcodebuild -version + OUTPUT_VARIABLE XCODE_VERSION + RESULT_VARIABLE XCODE_VERSION_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT ${XCODE_VERSION_RESULT}) + string(REGEX MATCH "Xcode [0-9\\.]+" XCODE_VERSION "${XCODE_VERSION}") + string(REGEX REPLACE "Xcode ([0-9\\.]+)" "\\1" XCODE_VERSION "${XCODE_VERSION}") + message(STATUS "Building with Xcode version: ${XCODE_VERSION}") +else() + message(FATAL_ERROR "Cannot execute xcodebuild, please check whether xcode is installed.") +endif() + +# Required as of cmake 2.8.10 +set(CMAKE_OSX_DEPLOYMENT_TARGET "" CACHE STRING "Force unset of the deployment target for iOS" FORCE) + +# Setup iOS platform unless specified manually with IOS_PLATFORM +if(NOT DEFINED IOS_PLATFORM) + set(IOS_PLATFORM "OS") +endif() +set(IOS_PLATFORM ${IOS_PLATFORM} CACHE STRING "Type of iOS Platform") + +# Set the architecture for iOS +if(NOT DEFINED IOS_ARCH) + if(IOS_PLATFORM STREQUAL "OS") + # FIXME(liuyiqun): support "armv7;armv7s;arm64" future + set(IOS_ARCH "arm64") + elseif(IOS_PLATFORM STREQUAL "SIMULATOR") + set(IOS_ARCH "i386;x86_64") + elseif(IOS_PLATFORM STREQUAL "WATCHOS") + set(IOS_ARCH armv7k) + endif() +endif() +set(CMAKE_OSX_ARCHITECTURES ${IOS_ARCH} CACHE string "Build architecture for iOS") + +# Specify minimum iOS deployment version +if(NOT DEFINED IOS_DEPLOYMENT_TARGET) + set(IOS_DEPLOYMENT_TARGET "7.0") +endif() +set(IOS_DEPLOYMENT_TARGET ${IOS_DEPLOYMENT_TARGET} CACHE STRING "Minimum iOS version") + +# Whether to enable bitcode +if(NOT DEFINED IOS_ENABLE_BITCODE) + set(IOS_ENABLE_BITCODE ON) +endif() +set(IOS_ENABLE_BITCODE ${IOS_ENABLE_BITCODE} CACHE BOOL "Whether to enable bitcode") + +if(NOT DEFINED IOS_USE_VECLIB_FOR_BLAS) + set(IOS_USE_VECLIB_FOR_BLAS OFF) +endif() +set(IOS_USE_VECLIB_FOR_BLAS ${IOS_UES_VECLIB_FOR_BLAS} CACHE BOOL "Whether to use veclib") + +# Check the platform selection and setup for developer root +if(${IOS_PLATFORM} STREQUAL "OS") + set(IOS_PLATFORM_LOCATION "iPhoneOS.platform") + set(XCODE_IOS_PLATFORM iphoneos) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-iphoneos") +elseif(${IOS_PLATFORM} STREQUAL "SIMULATOR") + set(IOS_PLATFORM_LOCATION "iPhoneSimulator.platform") + set(XCODE_IOS_PLATFORM iphonesimulator) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-iphonesimulator") +elseif(${IOS_PLATFORM} STREQUAL "WATCHOS") + set(IOS_PLATFORM_LOCATION "WatchOS.platform") + set(XCODE_IOS_PLATFORM watchos) + + # This causes the installers to properly locate the output libraries + set(CMAKE_XCODE_EFFECTIVE_PLATFORMS "-watchos") +else(${IOS_PLATFORM} STREQUAL "OS") + message(FATAL_ERROR "Unsupported IOS_PLATFORM value selected. Please set to\n" + "\t OS, SIMULATOR, or WATCHOS.") +endif() + +# Check iOS developer toolchain +if(NOT DEFINED IOS_DEVELOPER_ROOT) + # Setup iOS developer location + execute_process(COMMAND xcode-select -print-path + OUTPUT_VARIABLE XCODE_DEVELOPER_DIR + RESULT_VARIABLE XCODE_DEVELOPER_DIR_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + # Xcode 4.3 changed the installation location, choose the most recent one available + if(${XCODE_VERSION} VERSION_LESS "4.3.0") + set(IOS_DEVELOPER_ROOT "/Developer/Platforms/${IOS_PLATFORM_LOCATION}/Developer") + else() + set(IOS_DEVELOPER_ROOT "${XCODE_DEVELOPER_DIR}/Platforms/${IOS_PLATFORM_LOCATION}/Developer") + endif() +endif() +if(EXISTS ${IOS_DEVELOPER_ROOT}) + set(IOS_DEVELOPER_ROOT ${IOS_DEVELOPER_ROOT} CACHE PATH "Location of iOS Platform") +else() + message(FATAL_ERROR "Invalid IOS_DEVELOPER_ROOT: ${IOS_DEVELOPER_ROOT} does not exist.") +endif() + +# Check iOS SDK +if(NOT DEFINED IOS_SDK_ROOT) + # Find and use the most recent iOS sdk + file(GLOB IOS_SDK_LISTS "${IOS_DEVELOPER_ROOT}/SDKs/*") + if(IOS_SDK_LISTS) + list(SORT IOS_SDK_LISTS) + list(REVERSE IOS_SDK_LISTS) + list(GET IOS_SDK_LISTS 0 IOS_SDK_ROOT) + else(IOS_SDK_LISTS) + message(FATAL_ERROR "No iOS SDK's found in default search path ${IOS_DEVELOPER_ROOT}." + " Please manually set IOS_SDK_ROOT or install the iOS SDK.") + endif(IOS_SDK_LISTS) +endif() +if(EXISTS ${IOS_SDK_ROOT}) + set(IOS_SDK_ROOT ${IOS_SDK_ROOT} CACHE PATH "Location of the selected iOS SDK") + message(STATUS "iOS toolchain: ${IOS_SDK_ROOT}") +else() + message(FATAL_ERROR "Invalid IOS_SDK_ROOT: ${IOS_SDK_ROOT} does not exist.") +endif() + +# Set the sysroot default to the most recent SDK +set(CMAKE_OSX_SYSROOT ${IOS_SDK_ROOT} CACHE PATH "Sysroot used for iOS support") + +# Get version of iOS SDK +execute_process(COMMAND xcodebuild -sdk ${CMAKE_OSX_SYSROOT} -version SDKVersion + OUTPUT_VARIABLE IOS_SDK_VERSION + RESULT_VARIABLE IOS_SDK_VERSION_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) +if(${IOS_SDK_VERSION_RESULT}) + string(REGEX MATCH "(([0-9]+)\\.)+([0-9]+)" IOS_SDK_VERSION "${IOS_SDK_ROOT}") +endif() +if(NOT IOS_SDK_VERSION) + message(WARNING "Cannot get SDK's version.") + set(IOS_SDK_VERSION 1) +endif() +set(CMAKE_SYSTEM_VERSION ${IOS_SDK_VERSION}) + +# Find the C & C++ compilers for the specified SDK. +if(NOT CMAKE_C_COMPILER) + # Default to use clang + execute_process(COMMAND xcrun -sdk ${CMAKE_OSX_SYSROOT} -find clang + OUTPUT_VARIABLE IOS_C_COMPILER + RESULT_VARIABLE IOS_C_COMPILER_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(${IOS_C_COMPILER_RESULT}) + get_filename_component(IOS_C_COMPILER clang PROGRAM) + endif() +else(NOT CMAKE_C_COMPILER) + # User can set it in cmake command + get_filename_component(IOS_C_COMPILER ${CMAKE_C_COMPILER} PROGRAM) +endif(NOT CMAKE_C_COMPILER) +if(NOT EXISTS ${IOS_C_COMPILER}) + message(FATAL_ERROR "Cannot find C compiler: ${IOS_C_COMPILER}") +endif() + +if(NOT CMAKE_CXX_COMPILER) + # Default to use clang++ + execute_process(COMMAND xcrun -sdk ${CMAKE_OSX_SYSROOT} -find clang++ + OUTPUT_VARIABLE IOS_CXX_COMPILER + RESULT_VARIABLE IOS_CXX_COMPILER_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(${IOS_CXX_COMPILER_RESULT}) + get_filename_component(IOS_CXX_COMPILER clang++ PROGRAM) + endif() +else(NOT CMAKE_CXX_COMPILER) + # User can set it in cmake command + get_filename_component(IOS_CXX_COMPILER ${CMAKE_CXX_COMPILER} PROGRAM) +endif(NOT CMAKE_CXX_COMPILER) +if(NOT EXISTS ${IOS_CXX_COMPILER}) + message(FATAL_ERROR "Cannot find CXX compiler: ${IOS_CXX_COMPILER}") +endif() + +set(CMAKE_C_COMPILER ${IOS_C_COMPILER} CACHE PATH "C compiler" FORCE) +set(CMAKE_CXX_COMPILER ${IOS_CXX_COMPILER} CACHE PATH "CXX compiler" FORCE) + +set(CMAKE_C_OSX_COMPATIBILITY_VERSION_FLAG "-compatibility_version ") +set(CMAKE_C_OSX_CURRENT_VERSION_FLAG "-current_version ") +set(CMAKE_CXX_OSX_COMPATIBILITY_VERSION_FLAG "${CMAKE_C_OSX_COMPATIBILITY_VERSION_FLAG}") +set(CMAKE_CXX_OSX_CURRENT_VERSION_FLAG "${CMAKE_C_OSX_CURRENT_VERSION_FLAG}") + +# Set iOS specific C/C++ flags +if(IOS_PLATFORM STREQUAL "OS") + if(XCODE_VERSION VERSION_LESS "7.0") + set(XCODE_IOS_PLATFORM_VERSION_FLAGS "-mios-version-min=${IOS_DEPLOYMENT_TARGET}") + else() + # Xcode 7.0+ uses flags we can build directly from XCODE_IOS_PLATFORM. + set(XCODE_IOS_PLATFORM_VERSION_FLAGS "-m${XCODE_IOS_PLATFORM}-version-min=${IOS_DEPLOYMENT_TARGET}") + endif() +else() + set(XCODE_IOS_FLATFORM_VERSION_FLAGS "-mios-simulator-version-min=${IOS_DEPLOYMENT_TARGET}") +endif() + +if(IOS_ENABLE_BITCODE) + set(XCODE_IOS_BITCODE_FLAGS "${IOS_COMPILER_FLAGS} -fembed-bitcode") +else() + set(XCODE_IOS_BITCODE_FLAGS "") +endif() + +set(IOS_COMPILER_FLAGS "${XCODE_IOS_PLATFORM_VERSION_FLAGS} ${XCODE_IOS_BITCODE_FLAGS}") + +# Hidden visibilty is required for cxx on iOS +set(CMAKE_C_FLAGS "${IOS_COMPILER_FLAGS} ${CMAKE_C_FLAGS}" CACHE STRING "C flags") +set(CMAKE_CXX_FLAGS "${IOS_COMPILER_FLAGS} -fvisibility-inlines-hidden ${CMAKE_CXX_FLAGS}" CACHE STRING "CXX flags") + +set(IOS_LINK_FLAGS "${XCODE_IOS_PLATFORM_VERSION_FLAGS} -Wl,-search_paths_first") + +if(IOS_USE_VECLIB_FOR_BLAS) + # Find vecLib for iOS + set(VECLIB_SEARCH_DIRS + ${IOS_SDK_ROOT}/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks + ${IOS_SDK_ROOT}/System/Library/Frameworks/Accelerate.framework/Frameworks + ) + find_path(VECLIB_INC_DIR vecLib.h PATHS ${VECLIB_SEARCH_DIRS}/vecLib.framework/Headers) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(vecLib DEFAULT_MSG VECLIB_INC_DIR) + + if(VECLIB_FOUND) + if(VECLIB_INC_DIR MATCHES "^/System/Library/Frameworks/vecLib.framework.*") + set(IOS_LINK_FLAGS ${IOS_LINK_FLAGS} -lcblas "-framework vecLib") + message(STATUS "Found standalone vecLib.framework") + else() + set(IOS_LINK_FLAGS ${IOS_LINK_FLAGS} -lcblas "-framework Accelerate") + message(STATUS "Found vecLib as part of Accelerate.framework") + endif() + + endif() +endif() + +set(CMAKE_C_LINK_FLAGS "${IOS_LINK_FLAGS} ${CMAKE_C_LINK_FLAGS}") +set(CMAKE_CXX_LINK_FLAGS "${IOS_LINK_FLAGS} ${CMAKE_CXX_LINK_FLAGS}") + +set(CMAKE_PLATFORM_HAS_INSTALLNAME 1) +if(NOT IOS_ENABLE_BITCODE) + set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-dynamiclib -headerpad_max_install_names") + set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS "-bundle -headerpad_max_install_names") +else() + set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "-dynamiclib") + set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS "-bundle") +endif() +set(CMAKE_SHARED_MODULE_LOADER_C_FLAG "-Wl,-bundle_loader,") +set(CMAKE_SHARED_MODULE_LOADER_CXX_FLAG "-Wl,-bundle_loader,") +set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib" ".so" ".a") + +# hack: if a new cmake (which uses CMAKE_INSTALL_NAME_TOOL) runs on an old build tree +# (where install_name_tool was hardcoded) and where CMAKE_INSTALL_NAME_TOOL isn't in the cache +# and still cmake didn't fail in CMakeFindBinUtils.cmake (because it isn't rerun) +# hardcode CMAKE_INSTALL_NAME_TOOL here to install_name_tool, so it behaves as it did before, Alex +if(NOT DEFINED CMAKE_INSTALL_NAME_TOOL) + find_program(CMAKE_INSTALL_NAME_TOOL install_name_tool) +endif() + +# Set the find root to the iOS developer roots and to user defined paths +set(CMAKE_FIND_ROOT_PATH ${IOS_DEVELOPER_ROOT} ${IOS_SDK_ROOT} ${CMAKE_PREFIX_PATH} + CACHE string "iOS find search path root") + +# default to searching for frameworks first +set(CMAKE_FIND_FRAMEWORK FIRST) + +# set up the default search directories for frameworks +set(CMAKE_SYSTEM_FRAMEWORK_PATH + ${IOS_SDK_ROOT}/System/Library/Frameworks + ${IOS_SDK_ROOT}/System/Library/PrivateFrameworks + ${IOS_SDK_ROOT}/Developer/Library/Frameworks + ) + +# only search the iOS sdks, not the remainder of the host filesystem +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +message(STATUS "iOS: Targeting iOS '${CMAKE_SYSTEM_VERSION}', " + "building for '${IOS_PLATFORM}' platform, with architecture '${CMAKE_OSX_ARCHITECTURES}'") +message(STATUS "System CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") +message(STATUS "System CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}") + +# Used in ExternalProject command +string(REPLACE ";" "\\$" EXTERNAL_IOS_ARCHITECTURES "${CMAKE_OSX_ARCHITECTURES}") +set(EXTERNAL_OPTIONAL_ARGS + -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} + -DCMAKE_OSX_ARCHITECTURES=${EXTERNAL_IOS_ARCHITECTURES}) + +# This little macro lets you set any XCode specific property +macro(set_xcode_property TARGET XCODE_PROPERTY XCODE_VALUE) + set_property (TARGET ${TARGET} PROPERTY XCODE_ATTRIBUTE_${XCODE_PROPERTY} ${XCODE_VALUE}) +endmacro(set_xcode_property) + +# This macro lets you find executable programs on the host system +macro(find_host_package) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER) + set(IOS FALSE) + + find_package(${ARGN}) + + set(IOS TRUE) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endmacro(find_host_package) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 01a2f4d5fa357ca882162247cc52299a3d1d3030..957f8271e4841836956b0c3f2cf3d8c88a31192a 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -39,13 +39,14 @@ ExternalProject_Add( PREFIX ${GFLAGS_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DBUILD_TESTING=OFF - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_TESTING=OFF + -DCMAKE_BUILD_TYPE=Release + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=Release diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index b450a3016667dcb4ab229fe7ec8aaae8609d8171..b3fef738ccc0b5886bb0a32501bb7b7adade0ff1 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -34,16 +34,17 @@ ExternalProject_Add( PREFIX ${GLOG_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} - CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${GLOG_INSTALL_DIR}/lib - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DWITH_GFLAGS=ON - CMAKE_ARGS -Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags - CMAKE_ARGS -DBUILD_TESTING=OFF - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=${GLOG_INSTALL_DIR}/lib + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DWITH_GFLAGS=ON + -Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags + -DBUILD_TESTING=OFF + -DCMAKE_BUILD_TYPE=Release + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index e3970073a1a0b946fa1db6642799719d7a9fcf4f..6a2a79b7631b32e8a099797de509af64533bbb95 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -48,15 +48,16 @@ IF(WITH_TESTING) PREFIX ${GTEST_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR} - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DBUILD_GMOCK=ON - CMAKE_ARGS -Dgtest_disable_pthreads=ON - CMAKE_ARGS -Dgtest_force_shared_crt=ON - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_GMOCK=ON + -Dgtest_disable_pthreads=ON + -Dgtest_force_shared_crt=ON + -DCMAKE_BUILD_TYPE=Release + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=Release diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 4fc8d43fc10891603b79c01a1c769cae21c52655..143b57a954e4e6b2bf273535ebdf0fa8e3dab768 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -29,30 +29,41 @@ IF(NOT ${CBLAS_FOUND}) "${CBLAS_INSTALL_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE FILEPATH "openblas library." FORCE) - IF(APPLE) - SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}") - SET(COMMON_ARGS CC=${OPENBLAS_CC} NO_SHARED=1 NO_LAPACK=1 libs) - ELSE() - SET(COMMON_ARGS CC=${CMAKE_C_COMPILER} NO_SHARED=1 NO_LAPACK=1 libs) - ENDIF() + SET(OPENBLAS_CC "${CMAKE_C_COMPILER}") IF(CMAKE_CROSSCOMPILING) + SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER}) + GET_FILENAME_COMPONENT(CROSS_SUFFIX ${CMAKE_C_COMPILER} DIRECTORY) + SET(CROSS_SUFFIX ${CROSS_SUFFIX}/) IF(ANDROID) # arm_soft_fp_abi branch of OpenBLAS to support softfp # https://github.com/xianyi/OpenBLAS/tree/arm_soft_fp_abi SET(OPENBLAS_COMMIT "b5c96fcfcdc82945502a2303116a64d89985daf5") IF(ANDROID_ABI MATCHES "^armeabi(-v7a)?$") - SET(TARGET "ARMV7") + SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV7 ARM_SOFTFP_ABI=1 USE_THREAD=0) ELSEIF(ANDROID_ABI STREQUAL "arm64-v8a") - SET(TARGET "ARMV8") + SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV8 BINARY=64 USE_THREAD=0) + ENDIF() + ELSEIF(IOS) + # FIXME(liuyiqun): support multiple architectures + SET(OPENBLAS_COMMIT "b5c96fcfcdc82945502a2303116a64d89985daf5") + SET(OPENBLAS_CC "${OPENBLAS_CC} ${CMAKE_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + IF(CMAKE_OSX_ARCHITECTURES MATCHES "armv7") + SET(OPENBLAS_CC "${OPENBLAS_CC} -arch armv7") + SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV7 ARM_SOFTFP_ABI=1 USE_THREAD=0) + ELSEIF(CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + SET(OPENBLAS_CC "${OPENBLAS_CC} -arch arm64") + SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV8 BINARY=64 USE_THREAD=0 CROSS_SUFFIX=${CROSS_SUFFIX}) ENDIF() - SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=${TARGET} ARM_SOFTFP_ABI=1 USE_THREAD=0) ELSEIF(RPI) # use hardfp SET(OPENBLAS_COMMIT "v0.2.20") - SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=ARMV7 USE_THREAD=0) + SET(OPTIONAL_ARGS ${OPTIONAL_ARGS} TARGET=ARMV7 USE_THREAD=0) ENDIF() ELSE() + IF(APPLE) + SET(OPENBLAS_CC "${CMAKE_C_COMPILER} -isysroot ${CMAKE_OSX_SYSROOT}") + ENDIF() SET(OPENBLAS_COMMIT "v0.2.20") SET(OPTIONAL_ARGS "") IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^x86(_64)?$") @@ -60,6 +71,8 @@ IF(NOT ${CBLAS_FOUND}) ENDIF() ENDIF() + SET(COMMON_ARGS CC=${OPENBLAS_CC} NO_SHARED=1 NO_LAPACK=1 libs) + ExternalProject_Add( extern_openblas ${EXTERNAL_PROJECT_LOG_ARGS} diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index a887be2e2ae5e21562fc15c775bb24cc1553480e..7cf7ba85cca4c248dcc74e078124c0b3815ee380 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -173,7 +173,8 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}" "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}" "-Dprotobuf_WITH_ZLIB=ON" - "-DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}") + "-DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}" + ${EXTERNAL_OPTIONAL_ARGS}) SET(OPTIONAL_CACHE_ARGS "-DZLIB_ROOT:STRING=${ZLIB_ROOT}") ENDIF() diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 490c87d67ed79a238dd506127cd4d9855fab6626..46c68cce324f565ec9985ef1a280d6d933f88f1f 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -INCLUDE(ExternalProject) +IF(NOT WITH_PYTHON) + return() +ENDIF() + INCLUDE(python_module) FIND_PACKAGE(PythonInterp 2.7) -IF(WITH_PYTHON) - FIND_PACKAGE(PythonLibs 2.7) - # Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE. - ADD_LIBRARY(python SHARED IMPORTED GLOBAL) - SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES}) -ENDIF(WITH_PYTHON) +FIND_PACKAGE(PythonLibs 2.7) +# Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE. +ADD_LIBRARY(python SHARED IMPORTED GLOBAL) +SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES}) SET(py_env "") IF(PYTHONINTERP_FOUND) @@ -36,9 +37,5 @@ IF(PYTHONINTERP_FOUND) ENDIF() ENDIF(PYTHONINTERP_FOUND) -IF(WITH_PYTHON) - INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) - INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) -ELSE() - SET(PYTHON_LIBRARIES "") -ENDIF() +INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake index 744c766ee7b067058b2cb4aa7f7b761cbb9778d4..ce088ae7eaa3355f2f9761e8c421da0d7ef89fa7 100644 --- a/cmake/external/swig.cmake +++ b/cmake/external/swig.cmake @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +IF(NOT WITH_SWIG_PY) + return() +ENDIF() + FIND_PACKAGE(SWIG) IF(NOT SWIG_FOUND) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 2d7daed9bcd5b8d854ffae6dc1ea191d154c16fe..bb258c7b5581fc22b44f4fe15c119f8081f4767e 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -16,25 +16,14 @@ INCLUDE(ExternalProject) SET(WARPCTC_SOURCES_DIR ${THIRD_PARTY_PATH}/warpctc) SET(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc) -SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) -INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) - -SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" CACHE PATH "Warp-ctc Library Directory" FORCE) - -IF(WIN32) - SET(WARPCTC_LIBRARIES - "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) -ELSE(WIN32) - IF(APPLE) - SET(_warpctc_SHARED_SUFFIX dylib) - ELSE(APPLE) - SET(_warpctc_SHARED_SUFFIX so) - ENDIF(APPLE) - - SET(WARPCTC_LIBRARIES - "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.${_warpctc_SHARED_SUFFIX}" CACHE FILEPATH "Warp-ctc Library" FORCE) -ENDIF(WIN32) +SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" + CACHE PATH "Warp-ctc Directory" FORCE) +# Used in unit test test_WarpCTCLayer +SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" + CACHE PATH "Warp-ctc Library Directory" FORCE) +SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/libwarpctc${CMAKE_SHARED_LIBRARY_SUFFIX}" + CACHE FILEPATH "Warp-ctc Library" FORCE) IF(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) SET(USE_OMP OFF) @@ -49,22 +38,26 @@ ExternalProject_Add( PREFIX ${WARPCTC_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} - CMAKE_ARGS -DWITH_GPU=${WITH_GPU} - CMAKE_ARGS -DWITH_OMP=${USE_OMP} - CMAKE_ARGS -DWITH_TORCH=OFF - CMAKE_ARGS -DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON - CMAKE_ARGS -DBUILD_SHARED=ON - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} + -DWITH_GPU=${WITH_GPU} + -DWITH_OMP=${USE_OMP} + -DWITH_TORCH=OFF + -DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON + -DBUILD_SHARED=ON + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR} ) +MESSAGE(STATUS "warp-ctc library: ${WARPCTC_LIBRARIES}") +INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) + ADD_LIBRARY(warpctc STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET warpctc PROPERTY IMPORTED_LOCATION ${WARPCTC_LIBRARIES}) ADD_DEPENDENCIES(warpctc extern_warpctc) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index 5aecab90ca3cecdfdba0eac178a6ba07dfcb8745..c496a52b780364f3014f8fa3dfbc944a7aa7430e 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -34,15 +34,16 @@ ExternalProject_Add( GIT_TAG "v1.2.8" PREFIX ${ZLIB_SOURCES_DIR} UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${ZLIB_INSTALL_DIR} - CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DCMAKE_MACOSX_RPATH=ON - CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_INSTALL_PREFIX=${ZLIB_INSTALL_DIR} + -DBUILD_SHARED_LIBS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_MACOSX_RPATH=ON + -DCMAKE_BUILD_TYPE=Release + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ZLIB_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=Release diff --git a/cmake/flags.cmake b/cmake/flags.cmake index ff246b2eb4ed97dd14d45763569b661cefd203c8..4593ae6180b6d7deb61d897eb634b17ac0bb1683 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -128,8 +128,10 @@ set(GPU_COMMON_FLAGS ) if (APPLE) - # On Mac OS X build fat binaries with x86_64 architectures by default. - set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) + if(NOT CMAKE_CROSSCOMPILING) + # On Mac OS X build fat binaries with x86_64 architectures by default. + set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) + endif() else() set(GPU_COMMON_FLAGS -Wall diff --git a/cmake/system.cmake b/cmake/system.cmake index adf5e2c539740076ad1808353522c7467d765e64..396bd1a0797edea0522bb1f02349373563b7726a 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -24,11 +24,10 @@ IF(WIN32) SET(HOST_SYSTEM "win32") ELSE(WIN32) IF(APPLE) - EXEC_PROGRAM (sw_vers ARGS -productVersion OUTPUT_VARIABLE MACOSX_VERSION) - STRING(REGEX MATCH "[0-9]+.[0-9]+" VERSION "${MACOSX_VERSION}") - SET(MACOS_VERSION ${VERSION}) SET(HOST_SYSTEM "macosx") - IF(NOT DEFINED ENV{MACOSX_DEPLOYMENT_TARGET}) + EXEC_PROGRAM(sw_vers ARGS -productVersion OUTPUT_VARIABLE HOST_SYSTEM_VERSION) + STRING(REGEX MATCH "[0-9]+.[0-9]+" MACOS_VERSION "${HOST_SYSTEM_VERSION}") + IF(NOT DEFINED $ENV{MACOSX_DEPLOYMENT_TARGET}) # Set cache variable - end user may change this during ccmake or cmake-gui configure. SET(CMAKE_OSX_DEPLOYMENT_TARGET ${MACOS_VERSION} CACHE STRING "Minimum OS X version to target for deployment (at runtime); newer APIs weak linked. Set to empty string for default value.") @@ -49,6 +48,8 @@ ELSE(WIN32) ELSEIF(LINUX_ISSUE MATCHES "Fedora") SET(HOST_SYSTEM "fedora") ENDIF() + + STRING(REGEX MATCH "(([0-9]+)\\.)+([0-9]+)" HOST_SYSTEM_VERSION "${LINUX_ISSUE}") ENDIF(EXISTS "/etc/issue") IF(EXISTS "/etc/redhat-release") @@ -70,7 +71,7 @@ CMAKE_HOST_SYSTEM_INFORMATION(RESULT CPU_CORES QUERY NUMBER_OF_LOGICAL_CORES) MARK_AS_ADVANCED(HOST_SYSTEM CPU_CORES) -MESSAGE(STATUS "Found Paddle host system: ${HOST_SYSTEM}") +MESSAGE(STATUS "Found Paddle host system: ${HOST_SYSTEM}, version: ${HOST_SYSTEM_VERSION}") MESSAGE(STATUS "Found Paddle host system's CPU: ${CPU_CORES} cores") # configuration for cross-compiling @@ -82,6 +83,9 @@ IF(DEFINED CMAKE_SYSTEM_NAME) ELSEIF(${CMAKE_SYSTEM_NAME} STREQUAL "RPi") SET(RPI TRUE) INCLUDE(cross_compiling/raspberry_pi) + ELSEIF(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + SET(IOS TRUE) + INCLUDE(cross_compiling/ios) ENDIF() ENDIF() diff --git a/cmake/util.cmake b/cmake/util.cmake index 0da4969d310368ab27b0ed65237813c07d6e59f0..ac911052eb970c5a3e485e3178dd788b1517ca30 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -25,7 +25,9 @@ function(target_circle_link_libraries TARGET_NAME) endif() endforeach() if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") - list(APPEND LIBS "-undefined dynamic_lookup") + if(NOT IOS_ENABLE_BITCODE) + list(APPEND LIBS "-undefined dynamic_lookup") + endif() endif() list(REVERSE libsInArgn) target_link_libraries(${TARGET_NAME} diff --git a/doc/design/api.md b/doc/design/api.md index 8185d2af0ea264a2e7b4e28b9ed05279e4a22014..e6a4638d9100d9b07c3ee6b92b530a17eae1c162 100644 --- a/doc/design/api.md +++ b/doc/design/api.md @@ -3,7 +3,7 @@ ## Ingredients As our design principle is starting from the essence: how could we -allow users to express and solve their problems at neural networks. +allow users to express and solve their problems as neural networks. Some essential concepts that our API have to provide include: 1. A *topology* is an expression of *layers*. @@ -233,7 +233,7 @@ paddle.dist_train(model, num_parameter_servers=15) ``` -The pseudo code if `paddle.dist_train` is as follows: +The pseudo code of `paddle.dist_train` is as follows: ```python def dist_train(topology, parameters, trainer, reader, ...): diff --git a/doc/design/auto_gradient_check.md b/doc/design/auto_gradient_check.md index 1f4d4ec16f7c395005e610751d95c10f5f3adf52..f9991541bc51c6e13ffce4e9cec60f73dc800121 100644 --- a/doc/design/auto_gradient_check.md +++ b/doc/design/auto_gradient_check.md @@ -1,17 +1,17 @@ ## Auto Gradient Checker Design ## Backgraound： -- Operator forward computing is easy to check if the result is right because it has a clear definition. **But** backpropagation is a notoriously difficult algorithm to debug and get right: - - 1. you should get the right backpropagation formula according to the forward computation. - - 2. you should implement it right in CPP. - - 3. it's difficult to prepare test data. +- Generally, it is easy to check whether the forward computation of an Operator is correct or not. However, backpropagation is a notoriously difficult algorithm to debug and get right: + 1. you should get the right backpropagation formula according to the forward computation. + 2. you should implement it right in CPP. + 3. it's difficult to prepare test data. -- Auto gradient check gets a numeric gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: - - 1. numeric gradient checker only need forward operator. - - 2. user only need to prepare the input data for forward Operator. +- Auto gradient checking gets a numerical gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: + 1. numerical gradient checker only need forward operator. + 2. user only need to prepare the input data for forward Operator. ## Mathematical Theory -The following two document from stanford has a detailed explanation of how to get numeric gradient and why it's useful. +The following two document from Stanford has a detailed explanation of how to get numerical gradient and why it's useful. - [Gradient checking and advanced optimization(en)](http://deeplearning.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization) - [Gradient checking and advanced optimization(cn)](http://ufldl.stanford.edu/wiki/index.php/%E6%A2%AF%E5%BA%A6%E6%A3%80%E9%AA%8C%E4%B8%8E%E9%AB%98%E7%BA%A7%E4%BC%98%E5%8C%96) @@ -20,7 +20,7 @@ The following two document from stanford has a detailed explanation of how to ge ## Numeric Gradient Implementation ### Python Interface ```python -def get_numeric_gradient(op, +def get_numerical_gradient(op, input_values, output_name, input_to_check, @@ -30,13 +30,13 @@ def get_numeric_gradient(op, Get Numeric Gradient for an operator's input. :param op: C++ operator instance, could be an network - :param input_values: The input variables. Should be an dictionary, key is - variable name. Value is numpy array. + :param input_values: The input variables. Should be an dictionary, whose key is + variable name, and value is numpy array. :param output_name: The final output variable name. - :param input_to_check: The input variable need to get gradient. + :param input_to_check: The input variable with respect to which to compute the gradient. :param delta: The perturbation value for numeric gradient method. The smaller delta is, the more accurate result will get. But if that delta is - too small, it could occur numerical stability problem. + too small, it will suffer from numerical stability problem. :param local_scope: The local scope used for get_numeric_gradient. :return: The gradient array in numpy format. """ @@ -45,28 +45,28 @@ def get_numeric_gradient(op, ### Explaination: - Why need `output_name` - - One Operator may have multiple Output, you can get independent gradient from each Output. So user should set one output to calculate. + - An Operator may have multiple Output, one can get independent gradient from each Output. So caller should specify the name of the output variable. - Why need `input_to_check` - - One operator may have multiple inputs. Gradient Op can calculate the gradient of these Inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. + - One operator may have multiple inputs. Gradient Op can calculate the gradient of these inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. ### Core Algorithm Implementation ```python - # we only compute gradient of one element each time. - # we use a for loop to compute the gradient of every element. + # we only compute gradient of one element a time. + # we use a for loop to compute the gradient of each element. for i in xrange(tensor_size): - # get one input element throw it's index i. + # get one input element by its index i. origin = tensor_to_check.get_float_element(i) - # add delta to it, run op and then get the sum of the result tensor. + # add delta to it, run op and then get the new value of the result tensor. x_pos = origin + delta tensor_to_check.set_float_element(i, x_pos) y_pos = get_output() - # plus delta to this element, run op and get the sum of the result tensor. + # plus delta to this element, run op and get the new value of the result tensor. x_neg = origin - delta tensor_to_check.set_float_element(i, x_neg) y_neg = get_output() @@ -85,15 +85,15 @@ def get_numeric_gradient(op, Each Operator Kernel has three kinds of Gradient: -- 1. Numeric Gradient -- 2. CPU Operator Gradient -- 3. GPU Operator Gradient(if supported) +1. Numerical gradient +2. CPU kernel gradient +3. GPU kernel gradient (if supported) -Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as the reference value. +The numerical gradient only relies on forward Operator. So we use the numerical gradient as the reference value. And the gradient checking is performed in the following three steps: -- 1. calculate the numeric gradient. -- 2. calculate CPU kernel Gradient with the backward Operator and compare it with the numeric gradient. -- 3. calculate GPU kernel Gradient with the backward Operator and compare it with the numeric gradient.(if support GPU) +1. calculate the numerical gradient +2. calculate CPU kernel gradient with the backward Operator and compare it with the numerical gradient +3. calculate GPU kernel gradient with the backward Operator and compare it with the numeric gradient (if supported) #### Python Interface @@ -110,8 +110,8 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as :param forward_op: used to create backward_op :param input_vars: numpy value of input variable. The following computation will use these variables. - :param inputs_to_check: inputs var names that should check gradient. - :param output_name: output name that used to + :param inputs_to_check: the input variable with respect to which to compute the gradient. + :param output_name: The final output variable name. :param max_relative_error: The relative tolerance parameter. :param no_grad_set: used when create backward ops :param only_cpu: only compute and check gradient on cpu kernel. @@ -120,24 +120,24 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as ``` ### How to check if two numpy array is close enough? -if `abs_numeric_grad` is nearly zero, then use abs error for numeric_grad, not relative +if `abs_numerical_grad` is nearly zero, then use abs error for numerical_grad ```python -numeric_grad = ... +numerical_grad = ... operator_grad = numpy.array(scope.find_var(grad_var_name(name)).get_tensor()) -abs_numeric_grad = numpy.abs(numeric_grad) -# if abs_numeric_grad is nearly zero, then use abs error for numeric_grad, not relative +abs_numerical_grad = numpy.abs(numerical_grad) +# if abs_numerical_grad is nearly zero, then use abs error for numeric_grad, not relative # error. -abs_numeric_grad[abs_numeric_grad < 1e-3] = 1 +abs_numerical_grad[abs_numerical_grad < 1e-3] = 1 -diff_mat = numpy.abs(abs_numeric_grad - operator_grad) / abs_numeric_grad +diff_mat = numpy.abs(abs_numerical_grad - operator_grad) / abs_numerical_grad max_diff = numpy.max(diff_mat) ``` #### Notes： -1，The Input data for auto gradient checker should be reasonable to avoid numeric problem. +The Input data for auto gradient checker should be reasonable to avoid numerical stability problem. #### Refs: diff --git a/doc/design/functions_operators_layers.md b/doc/design/functions_operators_layers.md index d23ba56b5773a36d448a99e4abdebc1475ed789c..984b59f4c6971dfb6f46dfe342f2751f392c0e88 100644 --- a/doc/design/functions_operators_layers.md +++ b/doc/design/functions_operators_layers.md @@ -53,12 +53,12 @@ Let's explain using an example. Suppose that we are going to compose the FC usi ```python def operator.mul(X1, X2): O = Var() - paddle.cpp.create_operator("mul", input={X1, Y1], output=O) + paddle.cpp.create_operator("mul", input={X1, Y1}, output=O) return O def operator.add(X1, X2): O = Var() - paddle.cpp.create_operator("add", input={X1, X2], output=O) + paddle.cpp.create_operator("add", input={X1, X2}, output=O) return O ``` diff --git a/doc/design/graph.md b/doc/design/graph.md index 51b7f87638f8ddff752328a562fe0dd0fe56cfd1..7519a65df835a39fe14f6ef45530afff170191ff 100644 --- a/doc/design/graph.md +++ b/doc/design/graph.md @@ -56,7 +56,7 @@ For each parameter, like W and b created by `layer.fc`, marked as double circles ## Block and Graph -The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block[(https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. +The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block](https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. A Block keeps operators in an array `BlockDesc::ops` @@ -67,4 +67,4 @@ message BlockDesc { } ``` -in the order that there appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. +in the order that they appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. diff --git a/doc/design/parameters_in_cpp.md b/doc/design/parameters_in_cpp.md index b6f99bc7d9d6fafacb0a4bcff806b65d9aef98cc..a7ac3f17c44ca94a669a8f1e283b291bceb42317 100644 --- a/doc/design/parameters_in_cpp.md +++ b/doc/design/parameters_in_cpp.md @@ -1,19 +1,19 @@ # Design Doc: The C++ Class `Parameters` -`Parameters` is a concept we designed in Paddle V2 API. `Parameters` is a container of parameters, and make Paddle can shared parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). +`Parameters` is a concept we designed in PaddlePaddle V2 API. `Parameters` is a container of parameters, which makes PaddlePaddle capable of sharing parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). -We used Python to implement Parameters when designing V2 API before. There are several defects for current implementation: +We used Python to implement Parameters when designing V2 API before. There are several defects for the current implementation: * We just use `memcpy` to share Parameters between topologies, but this is very inefficient. -* We did not implement share Parameters while training. We just trigger `memcpy` when start training. +* We did not support sharing Parameters while training. We just trigger `memcpy` when start training. -It is necessary that we implement Parameters in CPP side. However, it could be a code refactoring for Paddle, because Paddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current Paddle implementation, there are three concepts associated with `Parameters`: +It is necessary that we implement Parameters in CPP side. However, it could result a code refactoring for PaddlePaddle, because PaddlePaddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current PaddlePaddle implementation, there are three concepts associated with `Parameters`: 1. `paddle::Parameter`. A `Parameters` is a container for `paddle::Parameter`. It is evident that we should use `paddle::Parameter` when developing `Parameters`. However, the `Parameter` class contains many functions and does not have a clear interface. It contains `create/store Parameter`, `serialize/deserialize`, `optimize(i.e SGD)`, `randomize/zero`. When we developing `Parameters`, we only use `create/store Parameter` functionality. -We should extract functionalities of Parameter into many classes to clean Paddle CPP implementation. +We should extract functionalities of Parameter into many classes to clean PaddlePaddle CPP implementation. 2. `paddle::GradientMachine` and its sub-classes, e.g., `paddle::MultiGradientMachine`, `paddle::NeuralNetwork`. We should pass `Parameters` to `paddle::GradientMachine` when `forward/backward` to avoid `memcpy` between topologies. @@ -24,7 +24,7 @@ Also, we should handle multi-GPU/CPU training, because `forward` and `backward` So `Parameters` should be used by `paddle::ParameterUpdater`, and `paddle::ParameterUpdater` should optimize `Parameters` (by SGD). -The step by step approach for implementation Parameters in Paddle C++ core is listed below. Each step should be a PR and could be merged into Paddle one by one. +The step by step approach for implementation Parameters in PaddlePaddle C++ core is listed below. Each step should be a PR and could be merged into PaddlePaddle one by one. 1. Clean `paddle::Parameter` interface. Extract the functionalities of `paddle::Parameter` to prepare for the implementation of Parameters. diff --git a/doc/design/program.md b/doc/design/program.md new file mode 100644 index 0000000000000000000000000000000000000000..fb8f86ac07af403c9fee015f2a3adbfaa3c6d631 --- /dev/null +++ b/doc/design/program.md @@ -0,0 +1,61 @@ +# Design Doc: ProgramDesc + +The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program. + +As described in [graph.md](./graph.md), the first five lines of the following PaddlePaddle program + +```python +x = layer.data("images") +l = layer.data("label") +y = layer.fc(x) +cost = layer.mse(y, l) +optimize(cost) +train(cost, reader=mnist.train()) +``` + +generates, or compiles, a PaddelPaddle program, which is represented by the following protobuf message: + +```protobuf +message ProgramDesc { + repeated BlockDesc blocks = 1; +} + +message BlockDesc { + required int32 parent = 1; + repeated VarDesc vars = 2; + repeated OpDesc ops = 3; +} + +message OpDesc { + AttrDesc attrs = 1; + ... +} + +message AttrDesc { + required AttrType type = 1; + + // index into ProgramDesc::blocks when type==BLOCK + optional int32 block = 2; + ... +} +``` + +When each of the first five lines runs, related Python function, e.g., `layer.fc`, calls C++ InferShape functions. This InferShape function needs to access the properties of VarDesc's accessed by the current OpDesc. These VarDesc's might not be defined in the current block, but in some ancestor blocks. This requires that we can trace the parent of a block. + +A nested block is often an attribute of an operator, most likely, an IfElseOp or a WhileOp. In above solution, all blocks are in `ProgramDesc::blocks`, this implicitly assigns a zero-based ID to each block -- the index of the block in `ProgramDesc::blocks`. So that `AttrDesc::block` could be an integer block ID. + +With this design, the InferShape function should take the following parameters: + +```c++ +void InferShape(int current_block, + int current_operator, + ProgramDesc* program // might change VarDesc values. + ) { + ... +} +``` + +where + +- `current_block` indices into `ProgramDesc::blocks`, +- `current_operator` indices into `BlockDesc::ops`. diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index f21f7af520df5171798326818ecb97c3bcd14a12..320dccec3ddc7bfe6042f4e65b2518ea7b1ad24a 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -52,7 +52,7 @@ Here are valid outputs: # a mini batch of three data items, each data item is a list (single column). [([1,1,1],), ([2,2,2],), -([3,3,3],), +([3,3,3],)] ``` Please note that each item inside the list must be a tuple, below is an invalid output: diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md new file mode 100644 index 0000000000000000000000000000000000000000..ad801ca421ca31c84b0a6b0a18d1d625c87e0de5 --- /dev/null +++ b/doc/design/refactorization.md @@ -0,0 +1,253 @@ +# Design Doc: Refactorization Overview + +The goal of refactorizaiton include: + +1. Make it easy for external contributors to write new elementory computaiton operations. +1. Make the codebase clean and readable. +1. Introduce a new design of computation representation -- a computation graph of operators and variables. +1. The graph representation helps implementing auto-scalable and auto fault recoverable distributed computing. + +## Computation Graphs + +1. PaddlePaddle represent the computation, training and inference of DL models, by computation graphs. + + 1. Please dig into [computation graphs](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/graph.md) for a solid example. + +1. Users write Python programs to describe the graphs and run it (locally or remotely). + +1. A graph is composed of *variables* and *operators*. + +1. The description of graphs must be able to be serialized/deserialized, so it + + 1. could to be sent to the cloud for distributed execution, and + 1. be sent to clients for mobile or enterprise deployment. + +1. The Python program do + + 1. *compilation*: runs a Python program to generate a protobuf message representation of the graph and send it to + 1. the C++ library `libpaddle.so` for local execution, + 1. the master process of a distributed training job for training, or + 1. the server process of a Kubernetes serving job for distributed serving. + 1. *execution*: according to the protobuf message, constructs instances of class `Variable` and `OperatorBase`, and run them. + +## Description and Realization + +At compile time, the Python program generates protobuf message representation of the graph, or the description of the graph. + +At runtime, the C++ program realizes the graph and run it. + +| | Representation (protobuf messages) | Realization (C++ class objects) | +|---|---|---| +|Data|[VarDesc](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L107)|[Variable](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h#L24)| +|Operation|[OpDesc](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L35)|[Operator](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L64)| +|Block|BlockDesc|Block| + +The word *graph* is exchangable with *block* in this document. A graph represent computation steps and local variables as a C++/Java program block, or a pair of { and }. + +## Compilation and Execution + +1. Run an applicaton Python program to describe the graph. In particular, + + 1. create VarDesc to represent local/intermediate variables, + 1. create operators and set attributes, + 1. validate attribute values, + 1. inference the type and the shape of variables, + 1. plan for memory-reuse for variables, + 1. generate backward and optimization part of the Graph. + 1. possiblly split the graph for distributed training. + +1. The invocation of `train` or `infer` in the application Python program: + + 1. create a new Scope instance in the [scope hierarchy](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/scope.md) for each run of a block, + 1. realize local variables defined in the BlockDesc message in the new scope, + 1. a scope is similar to the stack frame in programming languages, + + 1. create an instance of class `Block`, in which, + 1. realize operators in the BlockDesc message, + + 1. run the Block by calling + 1. `Block::Eval(vector* targets)` for forward and backward computations, or + 1. `Block::Eval(vector* targets)` for optimization. + + +## Intermediate Representation (IR) + +```text +Compile Time -> IR -> Runtime +``` + +### Benefit + +- Optimization + ```text + Compile Time -> IR -> Optimized IR -> Runtime + ``` +- Send automatically partitioned IR to different nodes. + - Automatic data parallel + ```text + Compile Time + |-> Single GPU IR + |-> [trainer-IR-0, trainer-IR-1, pserver-IR] + |-> Node-0 (runs trainer-IR-0) + |-> Node-1 (runs trainer-IR-1) + |-> Node-2 (runs pserver-IR) + ``` + - Automatic model parallel (planned for future) + +--- + +# Operator/OpWithKernel/OpKernel + +![class_diagram](http://api.paddlepaddle.org/graphviz?dot=https://gist.githubusercontent.com/reyoung/53df507f6749762675dff3e7ce53372f/raw/49caf1fb70820fb4a6c217634317c9306f361f36/op_op_with_kern_class_diagram.dot) + +--- + +# Operator +![class_diagram](http://api.paddlepaddle.org/graphviz?dot=https://gist.githubusercontent.com/reyoung/53df507f6749762675dff3e7ce53372f/raw/dd598e8f1976f5759f58af5e5ef94738a6b2e661/op.dot) + +* `Operator` is the fundamental building block as the user interface. + * Operator stores input/output variable name, and attributes. + * The `InferShape` interface is used to infer output variable shapes by its input shapes. + * Use `Run` to compute `input variables` to `output variables`. + +--- + +# OpWithKernel/Kernel + +![class_diagram](http://api.paddlepaddle.org/graphviz?dot=https://gist.githubusercontent.com/reyoung/53df507f6749762675dff3e7ce53372f/raw/9d7f4eba185cf41c8e2fbfb40ae21890dbddcd39/op_with_kernel.dot) + +* `OpWithKernel` inherits `Operator`. +* `OpWithKernel` contains a Kernel map. + * `OpWithKernel::Run` get device's kernel, and invoke `OpKernel::Compute`. + * `OpKernelKey` is the map key. Only device place now, but may be data type later. + +--- + +# Why separate Kernel and Operator + +* Separate GPU and CPU code. + * Make Paddle can run without GPU. +* Make one operator (which is user interface) can contain many implementations. + * Same mul op, different FP16, FP32 Kernel. different MKL, eigen kernel. +--- + +# Libraries for Kernel development + +* `Eigen::Tensor` contains basic math and element-wise functions. + * Note that `Eigen::Tensor` has broadcast implementation. + * Limit number of `tensor.device(dev) = ` in your code. +* `thrust::tranform` and `std::transform`. + * `thrust` has the same API as C++ standard library. Using `transform` can quickly implement a customized elementwise kernel. + * `thrust` has more complex API, like `scan`, `reduce`, `reduce_by_key`. +* Hand-writing `GPUKernel` and `CPU` code + * Do not write `.h`. CPU Kernel should be in `.cc`. GPU kernel should be in `.cu`. (`GCC` cannot compile GPU code.) +--- +# Operator Register + +## Why register is necessary? +We need a method to build mappings between Op type names and Op classes. + +## How to do the register? + +Maintain a map, whose key is the type name and value is corresponding Op constructor. + +--- +# The Registry Map + +### `OpInfoMap` + +`op_type(string)` -> `OpInfo` + +`OpInfo`: + +- **`creator`**: The Op constructor. +- **`grad_op_type`**: The type of the gradient Op. +- **`proto`**: The Op's Protobuf, including inputs, outputs and required attributes. +- **`checker`**: Used to check attributes. + +--- +# Related Concepts + +### Op_Maker +It's constructor takes `proto` and `checker`. They are compeleted during Op_Maker's construction. ([ScaleOpMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37)) + +### Register Macros +```cpp +REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, grad_op_class) +REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) +``` + +### `USE` Macros +make sure the registration process is executed and linked. + +--- +# Register Process +1. Write Op class, as well as its gradient Op class if there is. +2. Write Op maker class. In the constructor, describe its inputs, outputs, and attributes. +3. Invoke macro `REGISTER_OP`. The macro will + 1. call maker class to complete `proto` and `checker` + 2. with the completed `proto` and `checker`, build a new key-value pair in the `OpInfoMap` + +4. Invoke `USE` macro in where the Op is used to make sure it is linked. + +--- +# Backward Module (1/2) +### Create Backward Operator +- Mapping from forwarding Op to backward Op +![backward](https://gist.githubusercontent.com/dzhwinter/a6fbd4623ee76c459f7f94591fd1abf0/raw/61026ab6e518e66bde66a889bc42557a1fccff33/backward.png) + +--- +# Backward Module (2/2) +### Build Backward Network +- **Input** graph of forwarding operators +- **Output** graph of backward operators +- **corner case in construction** + - shared variable => insert `Add` operator + - no gradient => insert `fill_zero_grad` operator + - recursive netOp => call `Backward` recursively + - RNN Op => recursively call `Backward` on stepnet + + +--- +# Scope, Variable, Tensor + +* `Tensor` is an n-dimension array with type. + * Only dims and data pointers are stored in `Tensor`. + * All operators on `Tensor` is written in `Operator` or global functions. + * variable length Tensor design [LoDTensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/lod_tensor.md) +* `Variable` is the inputs and outputs of an operator. Not just `Tensor`. + * step_scopes in RNN is a variable and not a tensor. +* `Scope` is where variables store at. + * map + * `Scope` has a hierarchical structure. The local scope can get variable from its parent scope. + +--- +# Block (in design) +## the difference with original RNNOp +- as an operator is more intuitive than `RNNOp`, +- offers new interface `Eval(targets)` to deduce the minimal block to `Run`, +- fits the compile-time/ runtime separation design. + - during the compilation, `SymbolTable` stores `VarDesc`s and `OpDesc`s and serialize to a `BlockDesc` + - when graph executes, a Block with `BlockDesc` passed in creates `Op` and `Var` then `Run` + +--- +# Milestone +- take Paddle/books as the main line, the requirement of the models motivates framework refactoring, +- model migration + - framework development gives **priority support** to model migration, for example, + - the MNIST demo needs a Python interface, + - the RNN models require the framework to support `LoDTensor`. + - determine some timelines, + - heavily-relied Ops need to be migrated first, + - different models can be migrated parallelly. +- improve the framework at the same time +- accept imperfection, concentrated on solving the specific problem at the right price. + +--- +# Control the migration quality +- compare the performance of migrated models with old ones. +- follow google C style +- build the automatic workflow of generating Python/C++ documentations + - the documentation of layers and ops should be written inside the code + - take the documentation quality into account when doing PR + - preview the documentations, read and improve them from users' perspective diff --git a/doc/design/releasing_process.md b/doc/design/releasing_process.md index 0c10e782808ca6456347ec54cb5e921162731ede..62ff8f3229bbbb5bc82e4da29259baffc30c2c87 100644 --- a/doc/design/releasing_process.md +++ b/doc/design/releasing_process.md @@ -1,8 +1,8 @@ -# Paddle发行规范 +# PaddlePaddle发行规范 -Paddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示Paddle版本号。 +PaddlePaddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示PaddlePaddle版本号。 -Paddle每次发新的版本，遵循以下流程: +PaddlePaddle每次发新的版本，遵循以下流程: 1. 从`develop`分支派生出新的分支，分支名为`release/版本号`。例如，`release/0.10.0` 2. 将新分支的版本打上tag，tag为`版本号rc.Patch号`。第一个tag为`0.10.0rc1`，第二个为`0.10.0rc2`，依次类推。 @@ -27,14 +27,14 @@ Paddle每次发新的版本，遵循以下流程: 需要注意的是: -* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试Paddle的行为。 +* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试PaddlePaddle的行为。 * 在`release/版本号`分支存在的时候，如果有bugfix的行为，需要将bugfix的分支同时merge到`master`, `develop`和`release/版本号`这三个分支。 -# Paddle 分支规范 +# PaddlePaddle 分支规范 -Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 +PaddlePaddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 -* Paddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: +* PaddlePaddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: * `master`分支为稳定(stable branch)版本分支。每一个`master`分支的版本都是经过单元测试和回归测试的版本。 * `develop`分支为开发(develop branch)版本分支。每一个`develop`分支的版本都经过单元测试，但并没有经过回归测试。 * `release/版本号`分支为每一次Release时建立的临时分支。在这个阶段的代码正在经历回归测试。 @@ -42,18 +42,18 @@ Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branch * 其他用户的fork版本库并不需要严格遵守[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，但所有fork的版本库的所有分支都相当于特性分支。 * 建议，开发者fork的版本库使用`develop`分支同步主版本库的`develop`分支 * 建议，开发者fork的版本库中，再基于`develop`版本fork出自己的功能分支。 - * 当功能分支开发完毕后，向Paddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 + * 当功能分支开发完毕后，向PaddlePaddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 * 在评审过程中，开发者修改自己的代码，可以继续在自己的功能分支提交代码。 * BugFix分支也是在开发者自己的fork版本库维护，与功能分支不同的是，BugFix分支需要分别给主版本库的`master`、`develop`与可能有的`release/版本号`分支，同时提起`Pull Request`。 -# Paddle回归测试列表 +# PaddlePaddle回归测试列表 -本列表说明Paddle发版之前需要测试的功能点。 +本列表说明PaddlePaddle发版之前需要测试的功能点。 -## Paddle Book中所有章节 +## PaddlePaddle Book中所有章节 -Paddle每次发版本首先要保证Paddle Book中所有章节功能的正确性。功能的正确性包括验证Paddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 +PaddlePaddle每次发版本首先要保证PaddlePaddle Book中所有章节功能的正确性。功能的正确性包括验证PaddlePaddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 | | 新手入门章节 | 识别数字 | 图像分类 | 词向量 | 情感分析 | 语意角色标注 | 机器翻译 | 个性化推荐 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | diff --git a/doc/design/scope.md b/doc/design/scope.md index c9e0be716b606f6c7bf0373e0c6e632647e07a6f..b1f9bb4378eb5ec6926f1e53f7c1f4fd5674064c 100644 --- a/doc/design/scope.md +++ b/doc/design/scope.md @@ -17,7 +17,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope only contains a map of a name to variable. - All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state(momentum) etc. + All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state (momentum) etc. 1. Variable can only be created by Scope and a variable can only be got from Scope. User cannot create or get a variable outside a scope. This is a constraints of our framework, and will keep our framework simple and clear. @@ -32,7 +32,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope should destruct all Variables inside it when itself is destructed. User can never store `Variable` pointer somewhere else. - Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be a invalid pointer when associated `Scope` is destroyed. + Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be an invalid pointer when associated `Scope` is destroyed. ```cpp class Scope { @@ -50,7 +50,7 @@ class Scope { Just like [scope](https://en.wikipedia.org/wiki/Scope_(computer_science)) in programming languages, `Scope` in the neural network can also be a local scope. There are two attributes about local scope. -1. We can create local variables in a local scope. When that local scope are destroyed, all local variables should also be destroyed. +1. We can create local variables in a local scope. When that local scope is destroyed, all local variables should also be destroyed. 2. Variables in a parent scope can be retrieved from local scopes of that parent scope, i.e., when user get a variable from a scope, it will try to search this variable in current scope. If there is no such variable in the local scope, `scope` will keep searching from its parent, until the variable is found or there is no parent. ```cpp @@ -121,4 +121,4 @@ Also, as the parent scope is a `shared_ptr`, we can only `Create()` a scope shar ## Orthogonal interface -`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return a `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. +`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return an `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. diff --git a/doc/design/simple_op_design.md b/doc/design/simple_op_design.md index fded4a68612396a262121a5a886a8ae573dfa662..c7aeed7f9b4637e1c29d530f37b42d12500af82f 100644 --- a/doc/design/simple_op_design.md +++ b/doc/design/simple_op_design.md @@ -6,9 +6,9 @@ The Interaction between Python and C++ can be simplified as two steps: 1. C++ tells Python how many Ops there are, and what parameter do users need to offer to initialize a new Op. Python then builds API for each Op at compile time. -2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ fo finish Op construction task. +2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ for finishing the Op construction task. -### Message form C++ to Python +### Message from C++ to Python We define a Protobuf message class `OpProto` to hold message needed in the first step. What should an `OpProto` contain? This question is equivalent to “What message do we need to offer, to build a Python API which is legal and user oriented and can use to describe a whole Op.” @@ -193,7 +193,7 @@ def fc_layer(input, size, with_bias, activation): elif: # ... return act_output; -``` +``` ### Low Leval API diff --git a/doc/design/var_desc.md b/doc/design/var_desc.md index 86a95c10d5729704f86c285c9fe92db0cf2158be..bfbbdd0578ebc69ea4b49ade9b041573a9e9ad55 100644 --- a/doc/design/var_desc.md +++ b/doc/design/var_desc.md @@ -1,7 +1,7 @@ ## Background PaddlePaddle divides the description of neural network computation graph into two stages: compile time and runtime. -PaddlePaddle use proto message to describe compile time graph for +PaddlePaddle use proto message to describe compile time graph because 1. Computation graph should be able to be saved to a file. 1. In distributed training, the graph will be serialized and send to multiple workers. diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 138efb566e43fa71952f057829c2afbca96cadc9..00192aa69bd487787a8743d5589a365eacbd4ff3 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -321,3 +321,55 @@ pip uninstall py_paddle paddle 然后安装paddle的python环境, 在build目录下执行 pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl + +16. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 +--------------------------------------------------------- + +PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，请直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 + +将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时可以跳过PaddlePaddle模型参数文件的头信息。若在PaddlePaddle编译时，未指定按照double精度编译，默认情况下按照float精度计算，保存的参数也是float类型。这时在使用 :code:`numpy.array` 时，一般设置 :code:`dtype=float32` 。示例如下： + +.. code-block:: python + + def read_parameter(fname, width): + s = open(fname).read() + # skip header + vec = np.fromstring(s[16:], dtype=np.float32) + # width is the size of the corresponding layer + np.savetxt(fname + ".csv", vec.reshape(width, -1), + fmt="%.6f", delimiter=",") + + +将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面的代码将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 + +.. code-block:: python + + def gen_rand_param(param_file, width, height, need_trans): + np.random.seed() + header = struct.pack("iil", 0, 4, height * width) + param = np.float32(np.random.rand(height, width)) + with open(param_file, "w") as fparam: + fparam.write(header + param.tostring()) + +17. 如何加载预训练参数 +------------------------------ + +* 对加载预训练参数的层，设置其参数属性 :code:`is_static=True`，使该层的参数在训练过程中保持不变。以embedding层为例，代码如下： + +.. code-block:: python + + emb_para = paddle.attr.Param(name='emb', is_static=True) + paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) + + +* 从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。以embedding层为例，代码如下： + +.. code-block:: python + + def load_parameter(file_name, h, w): + with open(file_name, 'rb') as f: + f.read(16) # skip header. + return np.fromfile(f, dtype=np.float32).reshape(h, w) + + parameters = paddle.parameters.create(my_cost) + parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index c6570b89aedfaac1aef9b00e889b0b3ed21d8d65..264b998f50df016da0741d97d4b26f759ee90900 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -54,9 +54,9 @@ class MulOpMaker : public framework::OpProtoAndCheckerMaker { public: MulOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of mul op"); - AddInput("Y", "The second input of mul op"); - AddOutput("Out", "The output of mul op"); + AddInput("X", "(Tensor), 2D tensor of size (M x K)"); + AddInput("Y", "(Tensor), 2D tensor of size (K x N)"); + AddOutput("Out", "(Tensor), 2D tensor of size (M x N)"); AddComment(R"DOC( Two Element Mul Operator. The equation is: Out = X * Y @@ -72,7 +72,7 @@ The equation is: Out = X * Y 构造函数里通过`AddInput`添加输入参数，通过`AddOutput`添加输出参数，通过`AddComment`添加Op的注释。这些函数会将对应内容添加到`OpProto`中。 -上面的代码在`MulOp`中添加两个输入`X`和`Y`，添加了一个输出`Out`，并解释了各自含义，命名请遵守命名规范。 +上面的代码在`MulOp`中添加两个输入`X`和`Y`，添加了一个输出`Out`，并解释了各自含义，命名请遵守[命名规范](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/name_convention.md)。 再以[`ScaleOp`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37)为例： diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index ec866b2907d4623e8a94a249bc9af624071ade97..b435de80a224571d16efdee168541aa301c3f73a 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -19,7 +19,7 @@ if(Boost_FOUND) endif() if(WITH_C_API) - add_subdirectory(capi) + add_subdirectory(capi) endif() if(WITH_SWIG_PY) diff --git a/paddle/capi/CMakeLists.txt b/paddle/capi/CMakeLists.txt index 3af111eb5738c3f2f399ff4e5c06c8d2ecd8973e..dd9e4f1cbd636e29a6934d1119fc93ebc9d0ecee 100644 --- a/paddle/capi/CMakeLists.txt +++ b/paddle/capi/CMakeLists.txt @@ -28,42 +28,38 @@ add_style_check_target(paddle_capi ${CAPI_SOURCES} ${CAPI_HEADER} add_dependencies(paddle_capi paddle_proto) - # combine all paddle static libraries together, into libpaddle_capi_whole.a # user should use PaddleCAPI as -lpaddle_capi_whole -set(capi_whole_library libpaddle_capi_whole.a) -add_custom_target(paddle_capi_whole ALL - COMMAND mkdir -p o_files/capi && cd o_files/capi/ && ar -x $ - COMMAND mkdir -p o_files/utils && cd o_files/utils/ && ar -x $ - COMMAND mkdir -p o_files/parameter && cd o_files/parameter/ && ar -x $ - COMMAND mkdir -p o_files/math && cd o_files/math/ && ar -x $ - COMMAND mkdir -p o_files/cuda && cd o_files/cuda/ && ar -x $ - COMMAND mkdir -p o_files/function && cd o_files/function/ && ar -x $ - COMMAND mkdir -p o_files/gserver && cd o_files/gserver/ && ar -x $ - COMMAND mkdir -p o_files/proto && cd o_files/proto/ && ar -x $ - COMMAND mkdir -p o_files/network && cd o_files/network/ && ar -x $ - COMMAND mkdir -p o_files/pserver && cd o_files/pserver/ && ar -x $ - COMMAND ar crs ${capi_whole_library} `find ./o_files -name '*.o'` - COMMAND rm -rf o_files - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS paddle_capi paddle_utils paddle_parameter paddle_math - paddle_cuda paddle_function paddle_gserver - paddle_proto paddle_pserver paddle_network - ) -set_target_properties(paddle_capi_whole - PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/${capi_whole_library}) +set(PADDLE_CAPI_INFER_LIBS + paddle_utils + paddle_parameter + paddle_math + paddle_cuda + paddle_function + paddle_gserver + paddle_proto + paddle_pserver + paddle_network) + +cc_library(paddle_capi_whole DEPS paddle_capi ${PADDLE_CAPI_INFER_LIBS}) -set(LINK_FLAGS " -Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/export.sym -Wl,--version-script ${CMAKE_CURRENT_SOURCE_DIR}/export.map") -# TODO: merge mkl into paddle_capi_shared -add_library(paddle_capi_shared SHARED ${CAPI_SOURCES}) -set_target_properties(paddle_capi_shared PROPERTIES LINK_FLAGS "${LINK_FLAGS}") -target_include_directories(paddle_capi_shared PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) -link_paddle_exe(paddle_capi_shared) +# No shared library for iOS +if(NOT IOS) + set(LINK_FLAGS " -Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/export.sym -Wl,--version-script ${CMAKE_CURRENT_SOURCE_DIR}/export.map") + # TODO: merge mkl into paddle_capi_shared + add_library(paddle_capi_shared SHARED ${CAPI_SOURCES}) + set_target_properties(paddle_capi_shared PROPERTIES LINK_FLAGS "${LINK_FLAGS}") + target_include_directories(paddle_capi_shared PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + link_paddle_exe(paddle_capi_shared) +endif() # install library & headers. install(FILES ${CAPI_HEADERS} DESTINATION include/paddle) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/config.h DESTINATION include/paddle) if(ANDROID) + install(TARGETS paddle_capi_whole paddle_capi_shared + ARCHIVE DESTINATION lib/${ANDROID_ABI} + LIBRARY DESTINATION lib/${ANDROID_ABI}) execute_process( COMMAND ${GIT_EXECUTABLE} log --pretty=oneline -1 OUTPUT_VARIABLE GIT_COMMITS_LIST @@ -72,9 +68,6 @@ if(ANDROID) if(${GIT_COMMITS_LIST_RESULT}) set(GIT_COMMITS_LIST "No commits.") endif() - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${capi_whole_library} - DESTINATION lib/${ANDROID_ABI}) - install(TARGETS paddle_capi_shared DESTINATION lib/${ANDROID_ABI}) install(CODE "FILE(WRITE ${CMAKE_INSTALL_PREFIX}/lib/${ANDROID_ABI}/BUILD.txt \"Compiler:\n\" \"\\t${CMAKE_C_COMPILER}\\n\" @@ -88,8 +81,11 @@ if(ANDROID) )" ) else(ANDROID) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${capi_whole_library} DESTINATION lib) - install(TARGETS paddle_capi_shared DESTINATION lib) + install(TARGETS paddle_capi_whole + ARCHIVE DESTINATION lib) + if(NOT IOS) + install(TARGETS paddle_capi_shared DESTINATION lib) + endif() endif(ANDROID) # this variable used for unittest diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 3371962c635c3731f00a6af2a6e287ece33397cd..e535f84dba7c2726fbb70fa11ca8e9e2d29b8665 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -19,12 +19,14 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) proto_library(framework_proto SRCS framework.proto) cc_library(attribute SRCS attribute.cc DEPS framework_proto) +cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) +cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op) diff --git a/paddle/framework/attribute.cc b/paddle/framework/attribute.cc index 27132eaa0b3b0666fc042faf052dac2e169ba9e7..159ed03b92bbc57ab79734de832845ef1f367de9 100644 --- a/paddle/framework/attribute.cc +++ b/paddle/framework/attribute.cc @@ -19,6 +19,15 @@ limitations under the License. */ namespace paddle { namespace framework { +static ProgramDesc* g_program_desc = nullptr; + +ProgramDesc& GetProgramDesc() { + if (g_program_desc == nullptr) { + g_program_desc = new ProgramDesc(); + } + return *g_program_desc; +} + template <> AttrType AttrTypeID() { return INT; @@ -47,40 +56,44 @@ template <> AttrType AttrTypeID>>() { return INT_PAIRS; } +template <> +AttrType AttrTypeID() { + return BLOCK; +} Attribute GetAttrValue(const OpDesc::Attr& attr_desc) { switch (attr_desc.type()) { - case paddle::framework::AttrType::INT: { + case framework::AttrType::INT: { return attr_desc.i(); } - case paddle::framework::AttrType::FLOAT: { + case framework::AttrType::FLOAT: { return attr_desc.f(); } - case paddle::framework::AttrType::STRING: { + case framework::AttrType::STRING: { return attr_desc.s(); } - case paddle::framework::AttrType::INTS: { + case framework::AttrType::INTS: { std::vector val(attr_desc.ints_size()); for (int i = 0; i < attr_desc.ints_size(); ++i) { val[i] = attr_desc.ints(i); } return val; } - case paddle::framework::AttrType::FLOATS: { + case framework::AttrType::FLOATS: { std::vector val(attr_desc.floats_size()); for (int i = 0; i < attr_desc.floats_size(); ++i) { val[i] = attr_desc.floats(i); } return val; } - case paddle::framework::AttrType::STRINGS: { + case framework::AttrType::STRINGS: { std::vector val(attr_desc.strings_size()); for (int i = 0; i < attr_desc.strings_size(); ++i) { val[i] = attr_desc.strings(i); } return val; } - case paddle::framework::AttrType::INT_PAIRS: { + case framework::AttrType::INT_PAIRS: { std::vector> val(attr_desc.int_pairs_size()); for (int i = 0; i < attr_desc.int_pairs_size(); ++i) { val[i].first = attr_desc.int_pairs(i).first(); @@ -88,6 +101,9 @@ Attribute GetAttrValue(const OpDesc::Attr& attr_desc) { } return val; } + case framework::AttrType::BLOCK: { + return GetProgramDesc().mutable_blocks(attr_desc.block_idx()); + } } PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !"); return boost::blank(); diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index 2b788a76cafe198abb9aed8ba842e37cc6ff73a6..6735ca0a8d70d04dae9dd610742370e83472e9b0 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -29,11 +29,13 @@ namespace framework { typedef boost::variant, std::vector, std::vector, - std::vector>> + std::vector>, BlockDesc*> Attribute; typedef std::unordered_map AttributeMap; +ProgramDesc& GetProgramDesc(); + template AttrType AttrTypeID(); diff --git a/paddle/framework/ddim.cc b/paddle/framework/ddim.cc index fc3d508553c0e966978b28d58127bdbff10d45f1..a3357867530c110df16a5f3ec8c799735206cc71 100644 --- a/paddle/framework/ddim.cc +++ b/paddle/framework/ddim.cc @@ -292,5 +292,13 @@ DDim flatten_to_2d(const DDim& src, int num_col_dims) { DDim flatten_to_1d(const DDim& src) { return make_ddim({product(src)}); } +DDim stride(const DDim& ddim) { + std::vector strides(ddim.size()); + strides[ddim.size() - 1] = 1; + for (int i = ddim.size() - 2; i >= 0; --i) { + strides[i] = strides[i + 1] * ddim[i + 1]; + } + return framework::make_ddim(strides); +} } // namespace framework } // namespace paddle diff --git a/paddle/framework/ddim.h b/paddle/framework/ddim.h index ca29e7e8c7776de6adf3e3b0e8f11f0d4d8487c3..4a871bb0a91ed4050847509cc3f24218bcd57142 100644 --- a/paddle/framework/ddim.h +++ b/paddle/framework/ddim.h @@ -121,6 +121,7 @@ DDim flatten_to_2d(const DDim& src, int num_col_dims); DDim flatten_to_1d(const DDim& src); +DDim stride(const DDim& ddim); } // namespace framework } // namespace paddle diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index dfcb5fb6210a08f35193b83e3b5f7cee92f618d7..89a49f69062486ace67154f52450e7449a948851 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -23,6 +23,7 @@ enum AttrType { FLOATS = 4; STRINGS = 5; INT_PAIRS = 6; + BLOCK = 7; } message IntPair { @@ -44,6 +45,7 @@ message OpDesc { repeated float floats = 7; repeated string strings = 8; repeated IntPair int_pairs = 9; + optional int32 block_idx = 10; }; message Var { @@ -108,3 +110,12 @@ message VarDesc { required string name = 1; optional LoDTensorDesc lod_tensor = 2; } + +message BlockDesc { + required int32 idx = 1; + required int32 parent_idx = 2; + repeated VarDesc vars = 3; + repeated OpDesc ops = 4; +} + +message ProgramDesc { repeated BlockDesc blocks = 1; } diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 769b61f175a2f462258c1242d027c04c0abd12a9..07bbdf9416c432052b3222757a61ac4bfd70fe14 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -4,13 +4,13 @@ PaddlePaddle's RNN doesn't require that all instances have the same length. To ## Challenge of Variable-length Inputs -People usually represent a mini-batch by a Tensor. For example, a mini-batch of 32 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 32x32xO-dimensional tensor T and the 10x32x32 Tensor. +People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 10xOx32-dimensional tensor T and the 10x32x32 Tensor. Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths. ## LoD as a Solution -### Mini-Batch of variable-length sentenses +### Mini-Batch of variable-length sentences Let's imagine a mini-batch of 3 variable lengths sentences, containing 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information: @@ -51,17 +51,17 @@ The many 1's on the second level seem duplicated. For this particular case of 2 In summary, as long as that the essential elements (words or images) have the same size, we can represent mini-batches by a LoD Tensor: - The underlying tensor has size LxD1xD2x..., where D1xD2... is the size of the essential elements, and -- the first dimension size L has an additon property -- a LoD index as a nested vector: +- The first dimension size L has an additonal property -- a LoD index as a nested vector: ```c++ - typedef std::vector > LoD; + typedef std::vector> LoD; ``` -- The LoD index can is not necessary when there are only two levels and all elements of the second level have length 1. +- The LoD index is not necessary when there are only two levels and all elements of the second level have length 1. ## Slicing of LoD Tensor -Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 4 level LoD Tensor, for example, +Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 3 level LoD Tensor, for example, ``` 3 @@ -90,8 +90,9 @@ and the <1,2>-slice of above example is Let's go on slicing this slice. Its <1,1>-slice is ``` -3 -||| +1 +1 +| ``` ### The Slicing Algorithm @@ -99,7 +100,7 @@ Let's go on slicing this slice. Its <1,1>-slice is The algorithm, with over-simplified data structure, is defined as ```c++ -typedef vector > LoD; +typedef std::vector> LoD; struct LoDTensor { LoD lod_; @@ -128,7 +129,7 @@ Suppose that we want to retrieve the <1,2>-slice we will need to find out the starting position of this slice by summing over all leaf nodes in `LoD` to the left of the slice, i.e., 3 + 2 + 4 + 1 = 10. -To avoid the traversal of the LoD tree at slcing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into +To avoid the traversal of the LoD tree at slicing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into ``` 0 diff --git a/paddle/framework/op_proto_maker.cc b/paddle/framework/op_proto_maker.cc new file mode 100644 index 0000000000000000000000000000000000000000..151d61d5b175535509306d028027c7bc19abce81 --- /dev/null +++ b/paddle/framework/op_proto_maker.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +namespace paddle { +namespace framework { + +void OpProtoAndCheckerMaker::Validate() { + validated_ = true; + CheckNoDuplicatedInOutAttrs(); +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( + const std::string& name, const std::string& comment) { + auto* input = proto_->add_inputs(); + input->set_name(name); + input->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{input}; +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( + const std::string& name, const std::string& comment) { + auto* output = proto_->add_outputs(); + output->set_name(name); + output->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{output}; +} + +void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { + std::unordered_set names; + auto checker = [&](const std::string& name) { + PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); + names.insert(name); + }; + for (auto& attr : proto_->attrs()) { + checker(attr.name()); + } + for (auto& input : proto_->inputs()) { + checker(input.name()); + } + for (auto& output : proto_->outputs()) { + checker(output.name()); + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker.h b/paddle/framework/op_proto_maker.h new file mode 100644 index 0000000000000000000000000000000000000000..4d55a37db9f0a3deac7b3489c8bc288ea41f4799 --- /dev/null +++ b/paddle/framework/op_proto_maker.h @@ -0,0 +1,88 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/attribute.h" +#include "paddle/framework/framework.pb.h" + +namespace paddle { +namespace framework { + +// this class not only make proto but also init attribute checkers. +class OpProtoAndCheckerMaker { + public: + OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : proto_(proto), op_checker_(op_checker) {} + + virtual ~OpProtoAndCheckerMaker() { + PADDLE_ENFORCE(validated_, "should call Validate after build"); + } + + void Validate(); + + protected: + struct VariableBuilder { + OpProto::Var* var_; + + VariableBuilder& AsDuplicable() { + var_->set_duplicable(true); + return *this; + } + + VariableBuilder& AsIntermediate() { + var_->set_intermediate(true); + return *this; + } + + VariableBuilder& NotInGradient() { + var_->set_not_in_gradient(true); + return *this; + } + }; + + VariableBuilder AddInput(const std::string& name, const std::string& comment); + + VariableBuilder AddOutput(const std::string& name, + const std::string& comment); + + template + TypedAttrChecker& AddAttr(const std::string& name, + const std::string& comment, + bool generated = false) { + auto* attr = proto_->add_attrs(); + attr->set_name(name); + attr->set_comment(comment); + attr->set_generated(generated); + attr->set_type(AttrTypeID()); + return op_checker_->AddAttrChecker(name); + } + + void AddComment(const std::string& comment) { proto_->set_comment(comment); } + + private: + void CheckNoDuplicatedInOutAttrs(); + + OpProto* proto_; + OpAttrChecker* op_checker_; + bool validated_{false}; +}; + +class NOPMaker : public OpProtoAndCheckerMaker { + public: + NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) {} +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker_test.cc b/paddle/framework/op_proto_maker_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..b01e30f75371ca4aa63dae86ddfb966b1d4c7830 --- /dev/null +++ b/paddle/framework/op_proto_maker_test.cc @@ -0,0 +1,51 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +#include "gtest/gtest.h" + +class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestAttrProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddAttr("scale", "scale of test op"); + AddAttr("scale", "scale of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedAttr) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} + +class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestInOutProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of test op"); + AddInput("input", "input of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedInOut) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} \ No newline at end of file diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 572dff860a306bb03ba9e6702fec85e4a2ea1b54..90077d0192421f3678a049a723972fcb1e8d67af 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -24,6 +24,7 @@ limitations under the License. */ #include "paddle/framework/framework.pb.h" #include "paddle/framework/grad_op_builder.h" #include "paddle/framework/op_info.h" +#include "paddle/framework/op_proto_maker.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index c57537be4bf67a8db6a49669ab8d2ed1b1324bdc..49509af6630ada5c2ec724525ec0a6eab02679f9 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -22,14 +22,14 @@ namespace framework { template <> Eigen::DefaultDevice& ExecutionContext::GetEigenDevice< platform::CPUPlace, Eigen::DefaultDevice>() const { - return *device_context_->get_eigen_device(); + return *device_context_.get_eigen_device(); } #ifndef PADDLE_ONLY_CPU template <> Eigen::GpuDevice& ExecutionContext::GetEigenDevice() const { - return *device_context_->get_eigen_device(); + return *device_context_.get_eigen_device(); } #endif @@ -228,43 +228,5 @@ std::vector ExecutionContext::MultiOutput( return res; } -void OpProtoAndCheckerMaker::Validate() { - validated_ = true; - CheckNoDuplicatedInOutAttrs(); -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( - const std::string& name, const std::string& comment) { - auto* input = proto_->add_inputs(); - input->set_name(name); - input->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{input}; -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( - const std::string& name, const std::string& comment) { - auto* output = proto_->add_outputs(); - output->set_name(name); - output->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{output}; -} - -void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { - std::unordered_set names; - auto checker = [&](const std::string& name) { - PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); - names.insert(name); - }; - for (auto& attr : proto_->attrs()) { - checker(attr.name()); - } - for (auto& input : proto_->inputs()) { - checker(input.name()); - } - for (auto& output : proto_->outputs()) { - checker(output.name()); - } -} - } // namespace framework } // namespace paddle diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index adae7bfc3d7d31b1ed0373f01db4ef80343a08f7..1a78b6d1e146d2d157e353c5729d8518ee264517 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -167,71 +167,6 @@ class NOP : public OperatorBase { } }; -// this class not only make proto but also init attribute checkers. -class OpProtoAndCheckerMaker { - public: - OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) - : proto_(proto), op_checker_(op_checker) {} - - ~OpProtoAndCheckerMaker() { - PADDLE_ENFORCE(validated_, "should call Validate after build"); - } - - void Validate(); - - protected: - struct VariableBuilder { - OpProto::Var* var_; - - VariableBuilder& AsDuplicable() { - var_->set_duplicable(true); - return *this; - } - - VariableBuilder& AsIntermediate() { - var_->set_intermediate(true); - return *this; - } - - VariableBuilder& NotInGradient() { - var_->set_not_in_gradient(true); - return *this; - } - }; - - VariableBuilder AddInput(const std::string& name, const std::string& comment); - - VariableBuilder AddOutput(const std::string& name, - const std::string& comment); - - template - TypedAttrChecker& AddAttr(const std::string& name, - const std::string& comment, - bool generated = false) { - auto* attr = proto_->add_attrs(); - attr->set_name(name); - attr->set_comment(comment); - attr->set_generated(generated); - attr->set_type(AttrTypeID()); - return op_checker_->AddAttrChecker(name); - } - - void AddComment(const std::string& comment) { proto_->set_comment(comment); } - - private: - void CheckNoDuplicatedInOutAttrs(); - - OpProto* proto_; - OpAttrChecker* op_checker_; - bool validated_{false}; -}; - -class NOPMaker : public OpProtoAndCheckerMaker { - public: - NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) {} -}; - class InferShapeContext { public: InferShapeContext(const OperatorBase& op, const Scope& scope) @@ -366,7 +301,7 @@ struct EigenDeviceConverter { class ExecutionContext : public InferShapeContext { public: ExecutionContext(const OperatorBase& op, const Scope& scope, - const platform::DeviceContext* device_context) + const platform::DeviceContext& device_context) : InferShapeContext(op, scope), device_context_(device_context) {} template ::EigenDeviceType> DeviceType& GetEigenDevice() const; - platform::Place GetPlace() const { return device_context_->GetPlace(); } + platform::Place GetPlace() const { return device_context_.GetPlace(); } - const platform::DeviceContext* device_context() const { + const platform::DeviceContext& device_context() const { return device_context_; } @@ -401,7 +336,8 @@ class ExecutionContext : public InferShapeContext { return res; } - const platform::DeviceContext* device_context_; + private: + const platform::DeviceContext& device_context_; }; template <> @@ -461,7 +397,7 @@ class OperatorWithKernel : public OperatorBase { void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const final { auto& opKernel = AllOpKernels().at(type_).at(OpKernelKey(dev_ctx)); - opKernel->Compute(ExecutionContext(*this, scope, &dev_ctx)); + opKernel->Compute(ExecutionContext(*this, scope, dev_ctx)); } static std::unordered_map& diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 20bbb11896a4c6f11079669f0b25773f6460594d..0beab0fac5b94c78121261d2661a6f969289afc4 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -264,37 +264,3 @@ TEST(Operator, Clone) { auto b = a.Clone(); ASSERT_EQ(a.Type(), b->Type()); } - -class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestAttrProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddAttr("scale", "scale of test op"); - AddAttr("scale", "scale of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedAttr) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} - -class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestInOutProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("input", "input of test op"); - AddInput("input", "input of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedInOut) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} \ No newline at end of file diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index 2ba3f8ed355b48800cfa4180e4e8a94f2c9958a9..c93b03e48130afe9568089b6a7586c4185d1d5b4 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -58,6 +58,8 @@ class Scope { /// nullptr if cannot find. Variable* FindVar(const std::string& name) const; + const Scope& parent() const { return *parent_; } + /// Find the scope or an ancestor scope that contains the given variable. const Scope* FindScope(const Variable* var) const; diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index ed166935f76be9d25062b5e69536c7b7ac19045d..6d2c14f4c47afb755b1c74f6dc4dd10ab25ed191 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -130,15 +130,19 @@ inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const { PADDLE_ENFORCE_LE(end_idx, dims_[0], "Slice end index is out of bound."); PADDLE_ENFORCE_LT(begin_idx, end_idx, "Begin index must be less than end index."); - PADDLE_ENFORCE_NE(dims_[0], 1, "Can not slice a tensor with dims_[0] = 1."); - size_t base = numel() / dims_[0]; - Tensor dst; - dst.holder_ = holder_; - DDim dst_dims = dims_; - dst_dims[0] = end_idx - begin_idx; - dst.Resize(dst_dims); - dst.offset_ = offset_ + begin_idx * base * sizeof(T); - return dst; + + if (dims_[0] == 1) { + return *this; + } else { + size_t base = numel() / dims_[0]; + Tensor dst; + dst.holder_ = holder_; + DDim dst_dims = dims_; + dst_dims[0] = end_idx - begin_idx; + dst.Resize(dst_dims); + dst.offset_ = offset_ + begin_idx * base * sizeof(T); + return dst; + } } inline Tensor& Tensor::Resize(const DDim& dims) { diff --git a/paddle/function/neon/NeonDepthwiseConv.cpp b/paddle/function/neon/NeonDepthwiseConv.cpp index 18126152ea0b4ebfe4ec5c8084479787814ed173..38aa6670612b0771cdd8f1805a6d1bd9f281bdc1 100644 --- a/paddle/function/neon/NeonDepthwiseConv.cpp +++ b/paddle/function/neon/NeonDepthwiseConv.cpp @@ -52,7 +52,7 @@ public: int outputHeight = output[2]; int outputWidth = output[3]; int filterMultiplier = outputChannels / groups_; - CHECK_EQ(inputChannels, groups_); + CHECK_EQ(static_cast(inputChannels), groups_); // only support strideH() == strideW() and filterHeight == filterWidth. CHECK_EQ(strideH(), strideW()); diff --git a/paddle/gserver/activations/ActivationFunction.cpp b/paddle/gserver/activations/ActivationFunction.cpp index 78e958e06fac84fa956abc9faea60157bf6132eb..8b7b2e9b65898950e036ebc023cd28990cef303f 100644 --- a/paddle/gserver/activations/ActivationFunction.cpp +++ b/paddle/gserver/activations/ActivationFunction.cpp @@ -22,9 +22,12 @@ limitations under the License. */ #include #include "paddle/parameter/Argument.h" #include "paddle/utils/ClassRegistrar.h" - #include "paddle/utils/Logging.h" +#ifdef PADDLE_USE_MKLDNN +#include "MKLDNNActivation.h" +#endif + namespace paddle { static ClassRegistrar gActivationRegistrar; @@ -456,6 +459,12 @@ Error __must_check backward(Argument& act) { END_DEFINE_ACTIVATION(log) ActivationFunction* ActivationFunction::create(const std::string& type) { +#ifdef PADDLE_USE_MKLDNN + if (!type.empty() && type.compare(0, 7, "mkldnn_") == 0) { + return MKLDNNActivation::create(type); + } +#endif + return gActivationRegistrar.createByType(type); } diff --git a/paddle/gserver/activations/MKLDNNActivation.cpp b/paddle/gserver/activations/MKLDNNActivation.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ac50937ef3e28c1ac5aae651f9cf266ad07abcc4 --- /dev/null +++ b/paddle/gserver/activations/MKLDNNActivation.cpp @@ -0,0 +1,87 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MKLDNNActivation.h" +#include "mkldnn.hpp" +#include "paddle/utils/ClassRegistrar.h" + +namespace paddle { + +static ClassRegistrar gMKLDNNActivationRegistrar; +/** + * @def MKLDNN_ACTIVATION_CLASS_NAME + * @note MKLDNN_ACTIVATION_CLASS_NAME(relu) relu_; + * means mkldnn_reluActivation relu_; + */ +#define MKLDNN_ACTIVATION_CLASS_NAME(ACT_TYPE) mkldnn_##ACT_TYPE##Activation + +/** + * @def DEFINE_MKLDNN_ELTWISE_ACTIVATION + */ +#define DEFINE_MKLDNN_ELTWISE_ACTIVATION(ACT_TYPE, ALPHA, BWD_ALPHA) \ + class MKLDNN_ACTIVATION_CLASS_NAME(ACT_TYPE) \ + : public MKLDNNEltwiseActivation { \ + private: \ + static const std::string name; \ + static const float alpha; \ + static const float bwdAlpha; \ + \ + public: \ + const std::string& getName() const { return name; } \ + float getAlpha() const { return alpha; } \ + float getBwdAlpha() const { return bwdAlpha; } \ + }; \ + const std::string MKLDNN_ACTIVATION_CLASS_NAME(ACT_TYPE)::name = \ + "mkldnn_" #ACT_TYPE; \ + const float MKLDNN_ACTIVATION_CLASS_NAME(ACT_TYPE)::alpha = ALPHA; \ + const float MKLDNN_ACTIVATION_CLASS_NAME(ACT_TYPE)::bwdAlpha = BWD_ALPHA; \ + static InitFunction __reg_activation__mkldnn_##ACT_TYPE([] { \ + gMKLDNNActivationRegistrar \ + .registerClass( \ + "mkldnn_" #ACT_TYPE); \ + }); + +/** + * @brief MKLDNN Relu Activation. + * Actually mkldnn_relu is Leaky Relu. + * f(x) = x (x >= 0) + * f(x) = negative_slope * x (x < 0) + * @note the negative_slope should be -0.f in forward + */ +DEFINE_MKLDNN_ELTWISE_ACTIVATION(relu, -0.f, 0.f) + +/** + * @brief MKLDNN Tanh Activation. + */ +DEFINE_MKLDNN_ELTWISE_ACTIVATION(tanh, 0.f, 0.f) + +/** + * @brief MKLDNN ELU(Exponential Linear Unit) Activation. + * f(x) = x (x >= 0) + * f(x) = negative_slope * (exp(x) - 1) (x < 0) + */ +DEFINE_MKLDNN_ELTWISE_ACTIVATION(elu, 0.f, 0.f) + +ActivationFunction* MKLDNNActivation::create(const std::string& type) { + return gMKLDNNActivationRegistrar.createByType(type); +} + +std::vector MKLDNNActivation::getAllRegisteredTypes() { + std::vector types; + gMKLDNNActivationRegistrar.forEachType( + [&](const std::string& type) { types.push_back(type); }); + return types; +} + +} // namespace paddle diff --git a/paddle/gserver/activations/MKLDNNActivation.h b/paddle/gserver/activations/MKLDNNActivation.h new file mode 100644 index 0000000000000000000000000000000000000000..86ffe387366409d81a91740cc8cea886e618f7e2 --- /dev/null +++ b/paddle/gserver/activations/MKLDNNActivation.h @@ -0,0 +1,183 @@ +/* Copyright (c) 2017 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "ActivationFunction.h" +#include "mkldnn.hpp" +#include "paddle/gserver/layers/MKLDNNBase.h" +#include "paddle/math/MKLDNNMatrix.h" +#include "paddle/parameter/Argument.h" + +namespace paddle { + +/** + * @brief Base class of MKLDNN Activation. + * Common activation function are provieded, + * including mkldnn_relu, mkldnn_elu, mkldnn_tanh, mkldnn_softmax + */ +class MKLDNNActivation : public ActivationFunction { +protected: + // input value element count + size_t cnt_; + // should not merge the resetBwd into resetFwd, + // because the grad data would be changing before backward. + bool needResetBwd_; + // mkldnn matrix, primitive, stream and pipeline + MKLDNNMatrixPtr val_; + MKLDNNMatrixPtr grad_; + std::shared_ptr stream_; + std::shared_ptr fwd_; + std::shared_ptr bwd_; + std::vector pipelineFwd_; + std::vector pipelineBwd_; + +public: + MKLDNNActivation() : cnt_(0), needResetBwd_(true) {} + ~MKLDNNActivation() {} + static ActivationFunction* create(const std::string& type); + static std::vector getAllRegisteredTypes(); + virtual const std::string& getName() const = 0; + virtual Error __must_check forward(Argument& act) = 0; + virtual Error __must_check backward(Argument& act) = 0; +}; + +/** + * @brief Base class of MKLDNN Eltwise Activation, + * includes mkldnn_relu, mkldnn_elu and mkldnn_tanh. + */ +class MKLDNNEltwiseActivation : public MKLDNNActivation { + typedef mkldnn::eltwise_forward eltwise_fwd; + typedef mkldnn::eltwise_backward eltwise_bwd; + +protected: + // save the forward primitive desc, which can be used backward + std::shared_ptr fwdPD_; + // eltwise_bwd need src input value + MKLDNNMatrixPtr inVal_; + // use for copy data + std::shared_ptr copyInVal_; + +public: + MKLDNNEltwiseActivation() {} + + ~MKLDNNEltwiseActivation() {} + + virtual const std::string& getName() const = 0; + + // in common, the alpha of forward and backward should be equal. + // but for relu, to avoid negative value, they should be opposite + virtual float getAlpha() const = 0; + virtual float getBwdAlpha() const = 0; + virtual float getBeta() const { return 0.f; } + virtual mkldnn::algorithm getAlgo(const std::string& type) const { + if (type == "mkldnn_relu") { + return mkldnn::algorithm::eltwise_relu; + } else if (type == "mkldnn_tanh") { + return mkldnn::algorithm::eltwise_tanh; + } else if (type == "mkldnn_elu") { + return mkldnn::algorithm::eltwise_elu; + } else { + LOG(FATAL) << "Unkown eltwise activation type: " << type; + } + return (mkldnn::algorithm)0; + } + + /** + * reshape and reset the forward primitives + */ + void resetFwd(Argument& act) { + if (cnt_ == act.value->getElementCnt()) { + return; + } + cnt_ = act.value->getElementCnt(); + stream_.reset(new MKLDNNStream()); + auto eng = CPUEngine::Instance().getEngine(); + + // get algo setting + mkldnn::algorithm algo = getAlgo(this->getName()); + // note: alpha represents the NegativeSlope when used in relu. + float alpha = getAlpha(); + float beta = getBeta(); + + /// forward + pipelineFwd_.clear(); + val_ = std::dynamic_pointer_cast(act.value); + if (val_ == nullptr) { + int bs = act.getBatchSize(); + int ih = act.getFrameHeight() > 0 ? act.getFrameHeight() : 1; + int iw = act.getFrameWidth() > 0 ? act.getFrameWidth() : 1; + int ic = cnt_ / bs / ih / iw; + CHECK_EQ(cnt_, (size_t)bs * ic * ih * iw); + val_ = MKLDNNMatrix::create( + act.value, {bs, ic, ih, iw}, mkldnn::memory::format::nchw, eng); + CHECK(val_); + } + auto fwdDesc = eltwise_fwd::desc(mkldnn::prop_kind::forward_training, + algo, + val_->getMemoryDesc(), + alpha, + beta); + fwdPD_.reset(new eltwise_fwd::primitive_desc(fwdDesc, eng)); + // use inplace for forward but save input value before submit + inVal_ = val_; + copyInVal_ = nullptr; + if (act.grad && algo == mkldnn::algorithm::eltwise_tanh) { + // tanh need save src input for backward + inVal_ = MKLDNNMatrix::create(nullptr, val_->getPrimitiveDesc()); + copyInVal_ = std::make_shared(*val_, *inVal_); + CHECK(copyInVal_) << "should not be emptry"; + pipelineFwd_.push_back(*copyInVal_); + } + fwd_.reset(new eltwise_fwd(*fwdPD_, *val_, *val_)); + pipelineFwd_.push_back(*fwd_); + needResetBwd_ = true; + } + + /** + * reset the backward primitives, can not merge into resetFwd as the grad data + * would be changing before backward. + */ + void resetBwd(Argument& act) { + if (!needResetBwd_) { + return; + } + needResetBwd_ = false; + mkldnn::algorithm algo = getAlgo(this->getName()); + float alpha = getBwdAlpha(); + float beta = getBeta(); + grad_ = MKLDNNMatrix::create(act.grad, val_->getPrimitiveDesc()); + auto eng = CPUEngine::Instance().getEngine(); + auto bwdDesc = eltwise_bwd::desc( + algo, grad_->getMemoryDesc(), val_->getMemoryDesc(), alpha, beta); + auto bwdPD = eltwise_bwd::primitive_desc(bwdDesc, eng, *fwdPD_); + CHECK(inVal_); + bwd_.reset(new eltwise_bwd(bwdPD, *inVal_, *grad_, *grad_)); + pipelineBwd_.clear(); + pipelineBwd_.push_back(*bwd_); + } + + Error __must_check forward(Argument& act) { + resetFwd(act); + stream_->submit(pipelineFwd_); + return Error(); + } + + Error __must_check backward(Argument& act) { + resetBwd(act); + stream_->submit(pipelineBwd_); + return Error(); + } +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/DetectionOutputLayer.cpp b/paddle/gserver/layers/DetectionOutputLayer.cpp index 0cf0a92bf4bd8f9b8eba2016b2377d9dfb18c70a..f9040f7ae746f9ae1736cd477d3a69a2c49e9d34 100644 --- a/paddle/gserver/layers/DetectionOutputLayer.cpp +++ b/paddle/gserver/layers/DetectionOutputLayer.cpp @@ -143,7 +143,7 @@ void DetectionOutputLayer::forward(PassType passType) { resetOutput(numKept, 7); } else { MatrixPtr outV = getOutputValue(); - outV = NULL; + if (outV) outV->resize(0, 0); return; } MatrixPtr outV = getOutputValue(); diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index 2bc20eee6c452d0943dbf43b17ebe77976c97489..e95f42c863b3733ca66055e1b3939e734cae8ad1 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -14,26 +14,12 @@ limitations under the License. */ #include "paddle/utils/Util.h" +#include "CostLayer.h" +#include "ValidationLayer.h" #include "paddle/math/SparseMatrix.h" #include "paddle/utils/Error.h" #include "paddle/utils/Logging.h" -#include "AddtoLayer.h" -#include "CRFLayer.h" -#include "CosSimLayer.h" -#include "CostLayer.h" -#include "DataLayer.h" -#include "ExpandConvLayer.h" -#include "FullyConnectedLayer.h" -#include "HierarchicalSigmoidLayer.h" -#include "MaxLayer.h" -#include "MixedLayer.h" -#include "NormLayer.h" -#include "PoolLayer.h" -#include "TensorLayer.h" -#include "TransLayer.h" -#include "ValidationLayer.h" - DEFINE_bool(log_error_clipping, false, "enable log error clipping or not"); namespace paddle { @@ -109,6 +95,10 @@ ClassRegistrar Layer::registrar_; LayerPtr Layer::create(const LayerConfig& config) { std::string type = config.type(); + // NOTE: As following types have illegal character '-', + // they can not use REGISTER_LAYER to registrar. + // Besides, to fit with old training models, + // they can not use '_' instead. if (type == "multi-class-cross-entropy") return LayerPtr(new MultiClassCrossEntropy(config)); else if (type == "rank-cost") @@ -117,8 +107,6 @@ LayerPtr Layer::create(const LayerConfig& config) { return LayerPtr(new AucValidation(config)); else if (type == "pnpair-validation") return LayerPtr(new PnpairValidation(config)); - // NOTE: stop adding "if" statements here. - // Instead, use REGISTER_LAYER to add more layer types return LayerPtr(registrar_.createByType(config.type(), config)); } diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index 9088744beebd25ac105737fe3b012de143c66a7c..88b047c89bd40aba1afc456c22a2870c62989c1c 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -294,12 +294,9 @@ void MKLDNNConvLayer::resetOutValue( std::shared_ptr& pd, MKLDNNMatrixPtr& out) { out = MKLDNNMatrix::create(output_.value, pd->dst_primitive_desc()); - // change original output value from cpu matrix to mkldnn matrix - output_.value = std::dynamic_pointer_cast(out); - // create reorder if output value has cpu device and pd do not match cpuOutVal_ = nullptr; - cpuOutVal_ = nullptr; + cvtOutVal_ = nullptr; if (!outputIsOnlyMKLDNN()) { const MatrixPtr& cpuOut = getOutput(CPU_DEVICE).value; memory::dims outDims = memory::dims{bs_, oc_, oh_, ow_}; @@ -452,13 +449,14 @@ void MKLDNNConvLayer::resetOutGrad( cvtOutGrad_ = nullptr; if (!outputIsOnlyMKLDNN()) { const MatrixPtr& cpuOut = getOutput(CPU_DEVICE).grad; + outMat->setData(cpuOut->getData()); // same PrimitiveDesc with cpuInVal_ CHECK(cpuOutVal_); cpuOutGrad_ = MKLDNNMatrix::create(cpuOut, cpuOutVal_->getPrimitiveDesc()); if (cpuOutGrad_->getPrimitiveDesc() == out->getPrimitiveDesc()) { - outMat->setData(cpuOut->getData()); out = cpuOutGrad_; } else { + out = MKLDNNMatrix::create(nullptr, wgtPD->diff_dst_primitive_desc()); cvtOutGrad_ = MKLDNNMatrix::createReorder(cpuOutGrad_, out); CHECK(cvtOutGrad_); } diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index f60e221a6ec2ff513789a24e9f59bb25aef437b5..afd092666bf8b8a3389b36aa1f0edb256a9968e6 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -172,12 +172,10 @@ void MKLDNNFcLayer::resetWgtBiasValue(MKLDNNMatrixPtr& wgt, void MKLDNNFcLayer::resetOutValue(MKLDNNMatrixPtr& out) { out = MKLDNNMatrix::create(output_.value, {bs_, oc_}, format::nc, engine_); - // change original output value to mkldnn output value - output_.value = std::dynamic_pointer_cast(out); if (!outputIsOnlyMKLDNN()) { // fc cpu output value do not need create convert // just share point - getOutput(CPU_DEVICE).value->setData(output_.value->getData()); + getOutput(CPU_DEVICE).value->setData(out->getData()); } } @@ -234,6 +232,7 @@ void MKLDNNFcLayer::resetBwdBuffers(MKLDNNMatrixPtr& in, void MKLDNNFcLayer::resetOutGrad(MKLDNNMatrixPtr& out) { // TODO(TJ): merge outgrad int device = outputIsOnlyMKLDNN() ? MKLDNN_DEVICE : CPU_DEVICE; + output_.grad->setData(getOutput(device).grad->getData()); // for MKLDNN device: // can not directly cast outputgrad to mkldnnmatrix, // since each layer can not write the inputgrad to mkldnn inputgrad. diff --git a/paddle/gserver/layers/MKLDNNLayer.h b/paddle/gserver/layers/MKLDNNLayer.h index 169679c8297542cac4a43f5a8e1af311ad9282df..d8555a833187ddf64b096135e920e5be2b3a8c2f 100644 --- a/paddle/gserver/layers/MKLDNNLayer.h +++ b/paddle/gserver/layers/MKLDNNLayer.h @@ -119,6 +119,10 @@ public: inputElemenCnt_ = elemenCnt; reshape(bs_, ic_, ih_, iw_, oc_, oh_, ow_); resetFwd(pipelineFwd_, inVal_, wgtVal_, biasVal_, outVal_); + if (outVal_) { + // change original output value to mkldnn output value + output_.value = std::dynamic_pointer_cast(outVal_); + } convertWeightsFromPaddle(); needResetBwd_ = true; } @@ -137,18 +141,16 @@ public: } void backward(const UpdateCallback& callback) override { - /* Do derivation */ { + if (needResetBwd_) { + resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); + needResetBwd_ = false; + } + { REGISTER_TIMER_INFO("BpActTimer", getName().c_str()); backwardActivation(); } - { REGISTER_TIMER_INFO("mkldnn_bwdTimer", getName().c_str()); - if (needResetBwd_) { - resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); - needResetBwd_ = false; - } - stream_->submit(pipelineBwd_); } diff --git a/paddle/gserver/layers/MKLDNNPoolLayer.cpp b/paddle/gserver/layers/MKLDNNPoolLayer.cpp index 48b2f5a4cb37f6a9c4b1fdc6178c914b46c76e63..b62dfb7c54258a593aa50d5b30096423f375c69d 100644 --- a/paddle/gserver/layers/MKLDNNPoolLayer.cpp +++ b/paddle/gserver/layers/MKLDNNPoolLayer.cpp @@ -134,7 +134,6 @@ void MKLDNNPoolLayer::resetOutValue(MKLDNNMatrixPtr& out) { memory::dims outDims = memory::dims{bs_, oc_, oh_, ow_}; out = MKLDNNMatrix::create( output_.value, outDims, inVal_->getFormat(), engine_); - output_.value = std::dynamic_pointer_cast(out); // create reorder if output value has cpu device and pd do not match cpuOutVal_ = nullptr; diff --git a/paddle/gserver/layers/SequenceSliceLayer.cpp b/paddle/gserver/layers/SequenceSliceLayer.cpp index d3a83fad276a384ab3fddd5349912c56be6f3cc0..ce68ca449429711eeee692be750a4a2f1dac61a6 100644 --- a/paddle/gserver/layers/SequenceSliceLayer.cpp +++ b/paddle/gserver/layers/SequenceSliceLayer.cpp @@ -73,9 +73,10 @@ void SequenceSliceLayer::checkInputs() { CHECK(inputSeq.hasSeq()) << "The first input of sequence slice layer " << "must be a sequence."; const MatrixPtr indices1 = getInputValue(1); - CHECK_EQ(static_cast(indices1->getHeight()), - inputSeq.hasSubseq() ? inputSeq.getNumSubSequences() - : inputSeq.getNumSequences()) + CHECK_EQ( + indices1->getHeight(), + static_cast(inputSeq.hasSubseq() ? inputSeq.getNumSubSequences() + : inputSeq.getNumSequences())) << "Height of the second input should be equal to number of sequence " << "in the first input."; if (inputLayers_.size() == 3) { @@ -151,7 +152,7 @@ void SequenceSliceLayer::calSelectedRows(const MatrixPtr starts, if (ends) endPos = inputSeqInfoVec_[i][j] + ends->getElement(rowIdx, k); int seqLen = endPos - begPos + 1; - CHECK_GT(seqLen, 0U); + CHECK_GT(seqLen, 0); for (int m = begPos; m <= endPos; ++m) selectedRows_.push_back(m); hasSubseq ? outSubSeqStartPos_.push_back(outSubSeqStartPos_.back() + seqLen) diff --git a/paddle/gserver/tests/MKLDNNTester.cpp b/paddle/gserver/tests/MKLDNNTester.cpp index 2f48e5b2d3ffc9337ed1314f6db6549e56263fdd..f59618be9d09d146be52fb51cae84f4d24c15ef1 100644 --- a/paddle/gserver/tests/MKLDNNTester.cpp +++ b/paddle/gserver/tests/MKLDNNTester.cpp @@ -64,15 +64,17 @@ void MKLDNNTester::reset(const TestConfig& dnn, configs_[i], &(layerMaps_[i]), &(parameters_[i]), &(testLayers_[i])); } refLayer_ = testLayers_[REF]; - dnnLayer_ = std::dynamic_pointer_cast(testLayers_[DNN]); - CHECK(dnnLayer_); - // for comparison with Paddle reference results, - // need manually add cpu device output for test - dnnLayer_->addOutputArgument(CPU_DEVICE); + dnnLayer_ = testLayers_[DNN]; EXPECT_EQ(dataLayers_[DNN].size(), dataLayers_[REF].size()); EXPECT_EQ(parameters_[DNN].size(), parameters_[REF].size()); - setInputImgSize(); + + // for comparison with Paddle reference results, + // need manually add cpu device output for test + MKLDNNLayerPtr dnnLayer = std::dynamic_pointer_cast(dnnLayer_); + if (dnnLayer) { + dnnLayer->addOutputArgument(CPU_DEVICE); + } } void MKLDNNTester::setInputImgSize() { @@ -122,7 +124,7 @@ void MKLDNNTester::randomTopDiffs() { void MKLDNNTester::checkForward() { VLOG(MKLDNN_ALL) << "Check Forward"; printTopDatas(); - double delta = compareMatrix(dnnLayer_->getOutput(-1).value, + double delta = compareMatrix(dnnLayer_->getOutput(CPU_DEVICE).value, refLayer_->getOutputValue()); EXPECT_LE(fabs(delta), eps_); } @@ -155,7 +157,10 @@ void MKLDNNTester::checkBackwardWgts() { vector dnnWgts; // used to temply save mkldnn weights saveWgt(parameters_[DNN], dnnWgts); - dnnLayer_->convertWeightsToPaddle(); + MKLDNNLayerPtr dnnLayer = std::dynamic_pointer_cast(dnnLayer_); + if (dnnLayer) { + dnnLayer->convertWeightsToPaddle(); + } for (size_t i = 0; i < parameters_[DNN].size(); ++i) { const VectorPtr& dnn = parameters_[DNN][i]->getBuf(PARAMETER_VALUE); const VectorPtr& ref = parameters_[REF][i]->getBuf(PARAMETER_VALUE); @@ -322,6 +327,10 @@ void MKLDNNTester::runOnce() { // and clearTopDatas(REF) should be coverd by ref layers clearBotDiffs(REF); clearWgtDiffs(REF); + // it is necessary to clear bottom diffs when only activation is dnn type + if (configs_[DNN].layerConfig.active_type().compare(0, 7, "mkldnn_") == 0) { + clearBotDiffs(DNN); + } } void MKLDNNTester::run(const TestConfig& dnn, @@ -333,8 +342,19 @@ void MKLDNNTester::run(const TestConfig& dnn, float epsilon, bool log, int level) { - VLOG(MKLDNN_TESTS) << "Test MKLDNN functionality: " << dnn.layerConfig.type() - << " vs " << ref.layerConfig.type(); + CHECK(dnn.layerConfig.type().compare(0, 7, "mkldnn_") == 0 || + dnn.layerConfig.active_type().compare(0, 7, "mkldnn_") == 0) + << "should be MKLDNN layer or MKLDNN activation"; + if (dnn.layerConfig.type() == ref.layerConfig.type()) { + VLOG(MKLDNN_TESTS) << "Test MKLDNN functionality: " + << dnn.layerConfig.active_type() << " vs " + << ref.layerConfig.active_type(); + } else { + VLOG(MKLDNN_TESTS) << "Test MKLDNN functionality: " + << dnn.layerConfig.type() << " vs " + << ref.layerConfig.type(); + } + ih_ = inputImgH; iw_ = inputImgW; iter_ = iter; diff --git a/paddle/gserver/tests/MKLDNNTester.h b/paddle/gserver/tests/MKLDNNTester.h index 5ac885638cde7693a0c847733e7a6149c1b7e6c2..171d176ee757f1164c38d86273bdf9e5aefeda06 100644 --- a/paddle/gserver/tests/MKLDNNTester.h +++ b/paddle/gserver/tests/MKLDNNTester.h @@ -41,8 +41,7 @@ protected: vector layerMaps_; vector> parameters_; vector testLayers_; - LayerPtr refLayer_; - MKLDNNLayerPtr dnnLayer_; + LayerPtr refLayer_, dnnLayer_; /// run some iterations, all the result should pass size_t iter_; diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index b593f65fe49ef2271ad7cd0f609c9b828be03037..1bfbbde4246a10eaf86693a6a2f237f390966db3 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include #include "MKLDNNTester.h" #include "ModelConfig.pb.h" +#include "paddle/gserver/activations/MKLDNNActivation.h" #include "paddle/math/MathUtils.h" using namespace paddle; // NOLINT @@ -25,17 +26,26 @@ DECLARE_bool(thread_local_rand_use_global_seed); DECLARE_bool(use_gpu); DECLARE_bool(use_mkldnn); -struct testFCDesc { +#define RUN_MKLDNN_TEST(DNN_CONFIG, REF_CONFIG, DESC) \ + MKLDNNTester tester; \ + for (auto bs : {DESC.bs, 1}) { \ + tester.run(DNN_CONFIG, REF_CONFIG, bs, DESC.ih, DESC.iw); \ + } + +#define RUN_MKLDNN_TEST_LAYER(DNN_CONFIG, REF_TYPE, DESC) \ + TestConfig ref = DNN_CONFIG; \ + ref.layerConfig.set_type(REF_TYPE); \ + RUN_MKLDNN_TEST(DNN_CONFIG, ref, DESC) + +struct testFcDesc { int bs; int ic; int oc; int ih, iw; // oh == ow == 1 }; -void testFcLayer(const testFCDesc& pm) { - const std::string compareTypes[] = {"mkldnn_fc", "fc"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNFcConfig(TestConfig& cfg, const testFcDesc& pm) { + cfg.layerConfig.set_type("mkldnn_fc"); cfg.layerConfig.set_size(pm.oc); cfg.inputDefs.push_back( {INPUT_DATA, @@ -43,25 +53,25 @@ void testFcLayer(const testFCDesc& pm) { /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), /* size of weight= */ size_t(pm.oc * pm.ic * pm.ih * pm.iw)}); cfg.layerConfig.add_inputs(); +} - MKLDNNTester tester; +void testFcLayer(const testFcDesc& pm) { + TestConfig dnnConfig; + getMKLDNNFcConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "fc", pm) } } TEST(MKLDNNLayer, FcLayer) { - testFcLayer({/*bs*/ 2, /*ic*/ 2, /*oc*/ 3, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 3, /*ic*/ 7, /*oc*/ 19, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 8, /*ic*/ 16, /*oc*/ 32, /*ih*/ 13, /*iw*/ 13}); - testFcLayer({/*bs*/ 4, /*ic*/ 12, /*oc*/ 18, /*ih*/ 13, /*iw*/ 11}); - testFcLayer({/*bs*/ 2, /*ic*/ 64, /*oc*/ 32, /*ih*/ 16, /*iw*/ 16}); - testFcLayer({/*bs*/ 15, /*ic*/ 3, /*oc*/ 6, /*ih*/ 16, /*iw*/ 16}); + /* bs, ic, ih, iw, oc */ + testFcLayer({2, 2, 1, 1, 3}); + testFcLayer({3, 7, 1, 1, 19}); + testFcLayer({8, 16, 13, 13, 32}); + testFcLayer({4, 12, 13, 13, 18}); + testFcLayer({2, 64, 16, 16, 32}); + testFcLayer({15, 3, 16, 16, 6}); } struct testConvDesc { @@ -74,13 +84,10 @@ struct testConvDesc { int dh, dw; }; -void testConvLayer(const testConvDesc& pm) { - const std::string compareTypes[] = {"mkldnn_conv", "exconv"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNConvConfig(TestConfig& cfg, const testConvDesc& pm) { + cfg.layerConfig.set_type("mkldnn_conv"); cfg.layerConfig.set_num_filters(pm.oc); cfg.layerConfig.set_size(pm.oc * pm.oh * pm.ow); - // cfg.layerConfig.set_partial_sum(1); // TODO: check it cfg.layerConfig.set_shared_biases(true); cfg.inputDefs.push_back( {INPUT_DATA, @@ -114,15 +121,14 @@ void testConvLayer(const testConvDesc& pm) { int oh = outputSize(pm.ih, fh, pm.ph, pm.sh, true); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testConvLayer(const testConvDesc& pm) { + TestConfig dnnConfig; + getMKLDNNConvConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "exconv", pm) } } @@ -142,7 +148,7 @@ TEST(MKLDNNLayer, ConvLayer) { } struct testPoolDesc { - int bs, ch; // input channel and output channel are the same + int bs, ic; // input channel and output channel are the same int ih, iw; int oh, ow; int fh, fw; @@ -150,20 +156,18 @@ struct testPoolDesc { int sh, sw; }; -void testPoolLayer(const testPoolDesc& pm) { - const std::string compareTypes[] = {"mkldnn_pool", "pool"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); - cfg.layerConfig.set_size(pm.ch * pm.oh * pm.ow); +static void getMKLDNNPoolConfig(TestConfig& cfg, const testPoolDesc& pm) { + cfg.layerConfig.set_type("mkldnn_pool"); + cfg.layerConfig.set_size(pm.ic * pm.oh * pm.ow); cfg.inputDefs.push_back( {INPUT_DATA, "layer_0", - /* size of input layer= */ size_t(pm.ch * pm.ih * pm.iw), + /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), 0}); LayerInputConfig* input = cfg.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - // pool->set_pool_type(poolType); - pool->set_channels(pm.ch); + pool->set_pool_type("avg-projection"); + pool->set_channels(pm.ic); pool->set_img_size(pm.iw); pool->set_img_size_y(pm.ih); pool->set_output_x(pm.ow); @@ -179,20 +183,21 @@ void testPoolLayer(const testPoolDesc& pm) { int ow = outputSize(pm.iw, pm.fw, pm.pw, pm.sw, false); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testPoolLayer(const testPoolDesc& pm) { + TestConfig dnnConfig; + getMKLDNNPoolConfig(dnnConfig, pm); + LayerInputConfig* input = dnnConfig.layerConfig.mutable_inputs(0); + PoolConfig* pool = input->mutable_pool_conf(); for (auto type : {"max-projection", "avg-projection"}) { pool->set_pool_type(type); - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + RUN_MKLDNN_TEST_LAYER(dnnConfig, "pool", pm) } } -TEST(MkldnnLayer, PoolLayer) { - /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw*/ +TEST(MKLDNNLayer, PoolLayer) { + /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw */ testPoolLayer({2, 1, 4, 4, 2, 2, 3, 3, 0, 0, 2, 2}); testPoolLayer({10, 8, 16, 16, 8, 8, 2, 2, 0, 0, 2, 2}); testPoolLayer({4, 2, 5, 5, 3, 3, 3, 3, 1, 1, 2, 2}); @@ -203,6 +208,41 @@ TEST(MkldnnLayer, PoolLayer) { testPoolLayer({2, 8, 56, 56, 29, 29, 3, 3, 1, 1, 2, 2}); } +struct testActDesc { + int bs, ic, ih, iw; +}; + +static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { + cfg.biasSize = 0; + cfg.layerConfig.set_type("addto"); + size_t layerSize = pm.ih * pm.ih * pm.iw; + cfg.layerConfig.set_size(layerSize); + cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); + cfg.layerConfig.add_inputs(); +} + +void testActivation(std::string& actType, const testActDesc& pm) { + // TODO(TJ): mkldnn_softmax not implemented, paddle do not have elu activation + if (actType == "mkldnn_softmax" || actType == "mkldnn_elu") { + return; + } + const std::string compareTypes[] = {actType, actType.erase(0, 7)}; + TestConfig cfg; + getAddtoConfig(cfg, pm); + TestConfig ref = cfg; + cfg.layerConfig.set_active_type(compareTypes[0]); + ref.layerConfig.set_active_type(compareTypes[1]); + RUN_MKLDNN_TEST(cfg, ref, pm) +} + +TEST(MKLDNNActivation, Activations) { + auto types = MKLDNNActivation::getAllRegisteredTypes(); + for (auto type : types) { + /* bs, c, h, w*/ + testActivation(type, {16, 64, 32, 32}); + } +} + // TODO(TJ): add branch test int main(int argc, char** argv) { diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index e8ea6e37ac527a19c529d1731b94bed970211755..8193aa4adffc0409d8ea68417c68fa153a2942d8 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -26,7 +26,7 @@ limitations under the License. */ #include #endif -#ifdef PADDLE_USE_ATLAS +#if defined(PADDLE_USE_ATLAS) || defined(PADDLE_USE_VECLIB) extern "C" { #include #include diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index e3e934bcccd1a5f34d88a2f33f3708a46ddabe05..f8b0bce6815ff17a60ef64b0eec34a7cc9d16e72 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -55,6 +55,13 @@ function(op_library TARGET) set(pybind_flag 1) endif() + # activation_op contains several operators + if ("${TARGET}" STREQUAL "activation_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(sigmoid);\n") + endif() + # pybind USE_NO_KERNEL_OP file(READ ${TARGET}.cc TARGET_CONTENT) string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}") @@ -96,3 +103,4 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) +cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..cc55767cef9552475321bcb8c06d74a8d91dc99b --- /dev/null +++ b/paddle/operators/activation_op.cc @@ -0,0 +1,306 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/activation_op.h" + +namespace paddle { +namespace operators { + +class ActivationOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output("Y")->Resize( + ctx.Input("X")->dims()); + } +}; + +class ActivationOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + ctx.Output(framework::GradVarName("X")) + ->Resize(ctx.Input("Y")->dims()); + } +}; + +class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SigmoidOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sigmoid operator"); + AddOutput("Y", "Output of Sigmoid operator"); + AddComment("Sigmoid activation operator, sigmoid = 1 / (1 + exp(-x))"); + } +}; + +class ExpOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ExpOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Exp operator"); + AddOutput("Y", "Output of Exp operator"); + AddComment("Exp activation operator, exp(x) = e^x"); + } +}; + +class ReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Relu operator"); + AddOutput("Y", "Output of Relu operator"); + AddComment("Relu activation operator, relu(x) = max(x, 0)"); + } +}; + +class TanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Tanh operator"); + AddOutput("Y", "Output of Tanh operator"); + AddComment( + "Tanh activation operator, tanh = (exp(x) - exp(-x)) / (exp(x) + " + "exp(-x))"); + } +}; + +class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SqrtOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Sqrt operator"); + AddOutput("Y", "Output of Sqrt operator"); + AddComment("Sqrt activation operator, sqrt(x) = x^(1/2)"); + } +}; + +class AbsOpMaker : public framework::OpProtoAndCheckerMaker { + public: + AbsOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Abs operator"); + AddOutput("Y", "Output of Abs operator"); + AddComment("Abs activation operator, abs(x) = |x|"); + } +}; + +class ReciprocalOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ReciprocalOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Reciprocal operator"); + AddOutput("Y", "Output of Reciprocal operator"); + AddComment("Reciprocal activation operator, reciprocal(x) = 1 / x"); + } +}; + +class LogOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LogOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Log operator"); + AddOutput("Y", "Output of Log operator"); + AddComment("Log activation operator, log(x) = natural logarithm of x"); + } +}; + +class SquareOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SquareOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Square operator"); + AddOutput("Y", "Output of Square operator"); + AddComment("Square activation operator, square(x) = x^2"); + } +}; + +template +class BReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + BReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of BRelu operator"); + AddOutput("Y", "Output of BRelu operator"); + AddComment("BRelu activation operator, brelu = max(min(x, t_min), t_max)"); + AddAttr("t_min", "The min marginal value of BRelu") + .SetDefault(static_cast(0)); + AddAttr("t_max", "The max marginal value of BRelu") + .SetDefault(static_cast(24)); + } +}; + +template +class SoftReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SoftReluOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of SoftRelu operator"); + AddOutput("Y", "Output of SoftRelu operator"); + AddComment( + "SoftRelu activation operator, soft_relu = log(1 + exp(max(min(x, " + "threshold), threshold)))"); + AddAttr("threshold", "The threshold value of SoftRelu") + .SetDefault(static_cast(40)); + } +}; + +template +class PowOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PowOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of Pow operator"); + AddOutput("Y", "Output of Pow operator"); + AddComment("Pow activation operator, pow(x, factor) = x^factor"); + AddAttr("factor", "The exponential factor of Pow") + .SetDefault(static_cast(1)); + } +}; + +template +class STanhOpMaker : public framework::OpProtoAndCheckerMaker { + public: + STanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of STanh operator"); + AddOutput("Y", "Output of STanh operator"); + AddComment("STanh activation operator, stanh = b * tanh(a * x)"); + AddAttr("scale_a", "The scale parameter of a for the input") + .SetDefault(static_cast(2 / 3)); + AddAttr("scale_b", "The scale parameter of b for the input") + .SetDefault(static_cast(1.7159)); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(sigmoid, ops::ActivationOp, ops::SigmoidOpMaker, sigmoid_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP(exp, ops::ActivationOp, ops::ExpOpMaker, exp_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(exp_grad, + ops::ActivationGradKernel); + +REGISTER_OP(relu, ops::ActivationOp, ops::ReluOpMaker, relu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP(abs, ops::ActivationOp, ops::AbsOpMaker, abs_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP(reciprocal, ops::ActivationOp, ops::ReciprocalOpMaker, + reciprocal_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_CPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP(log, ops::ActivationOp, ops::LogOpMaker, log_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP(square, ops::ActivationOp, ops::SquareOpMaker, square_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_CPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_CPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, + soft_relu_grad, ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_CPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker, pow_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_CPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker, stanh_grad, + ops::ActivationOpGrad); +REGISTER_OP_CPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_CPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..feed1302b292a546f88fa35457c86aa2cfdaa307 --- /dev/null +++ b/paddle/operators/activation_op.cu @@ -0,0 +1,100 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/activation_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(sigmoid, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + sigmoid_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + exp, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(exp_grad, + ops::ActivationGradKernel); +REGISTER_OP_GPU_KERNEL(relu, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + relu_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + tanh, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + tanh_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + sqrt, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + sqrt_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + abs, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL(abs_grad, + ops::ActivationGradKernel); + +REGISTER_OP_GPU_KERNEL(reciprocal, + ops::ActivationKernel>); +REGISTER_OP_GPU_KERNEL( + reciprocal_grad, + ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL( + log, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + log_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(square, + ops::ActivationKernel); +REGISTER_OP_GPU_KERNEL( + square_grad, ops::ActivationGradKernel>); + +REGISTER_OP_GPU_KERNEL(brelu, + ops::BReluKernel); +REGISTER_OP_GPU_KERNEL(brelu_grad, + ops::BReluGradKernel); + +REGISTER_OP_GPU_KERNEL(soft_relu, + ops::SoftReluKernel); +REGISTER_OP_GPU_KERNEL( + soft_relu_grad, ops::SoftReluGradKernel); + +REGISTER_OP_GPU_KERNEL(pow, ops::PowKernel); +REGISTER_OP_GPU_KERNEL(pow_grad, + ops::PowGradKernel); + +REGISTER_OP_GPU_KERNEL(stanh, + ops::STanhKernel); +REGISTER_OP_GPU_KERNEL(stanh_grad, + ops::STanhGradKernel); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h new file mode 100644 index 0000000000000000000000000000000000000000..15f8afb4ba45cc989fe7576b82b8bf853b1df7de --- /dev/null +++ b/paddle/operators/activation_op.h @@ -0,0 +1,353 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class ActivationKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y); + } +}; + +template +class ActivationGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + Functor functor; + functor(place, x, y, dy, dx); + } +}; + +// sigmoid(x) = 1 / (1 + exp(-x)) +template +struct SigmoidFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / (static_cast(1) + (-x).exp()); + } +}; + +template +struct SigmoidGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y * (static_cast(1) - y); + } +}; + +// exp(x) = e^x +struct ExpFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.exp(); + } +}; + +struct ExpGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * y; + } +}; + +// relu(x) = max(x, 0) +template +struct ReluFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.cwiseMax(static_cast(0)); + } +}; + +template +struct ReluGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (x > static_cast(0)).template cast(); + } +}; + +// tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) +struct TanhFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.tanh(); + } +}; + +template +struct TanhGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) - y * y); + } +}; + +// sqrt(x) = x^(1/2) +struct SqrtFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.sqrt(); + } +}; + +template +struct SqrtGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + const Y y_conj = Eigen::numext::conj(y); + dx.device(d) = static_cast(0.5) * dy / y_conj; + } +}; + +// abs(x) = |x| +struct AbsFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.abs(); + } +}; + +struct AbsGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * x.sign(); + } +}; + +// reciprocal(x) = 1 / x +template +struct ReciprocalFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = static_cast(1) / x; + } +}; + +template +struct ReciprocalGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(-1) * y * y; + } +}; + +// log(x) = natural logarithm of x +struct LogFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.log(); + } +}; + +template +struct LogGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * (static_cast(1) / x); + } +}; + +// square(x) = x^2 +struct SquareFunctor { + template + void operator()(Device d, X x, Y y) { + y.device(d) = x.square(); + } +}; + +template +struct SquareGradFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) { + dx.device(d) = dy * static_cast(2) * x; + } +}; + +template +class BReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.cwiseMax(t_min).cwiseMin(t_max); + } +}; + +template +class BReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto t_min = static_cast(context.Attr("t_min")); + auto t_max = static_cast(context.Attr("t_max")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * ((x > t_min) * (x < t_max)).template cast(); + } +}; + +template +class SoftReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto threshold = static_cast(context.Attr("threshold")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + auto temp = x.cwiseMax(-threshold).cwiseMin(threshold).eval(); + y.device(place) = (static_cast(1) + temp.exp()).log(); + } +}; + +template +class SoftReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Input("Y"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto threshold = static_cast(context.Attr("threshold")); + dX->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto dy = framework::EigenVector::Flatten(*dY); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + auto temp = ((x > -threshold) * (x < threshold)).template cast().eval(); + dx.device(place) = dy * (static_cast(1) - (-y).exp()) * temp; + } +}; + +template +class PowKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto factor = static_cast(context.Attr("factor")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = x.pow(factor); + } +}; + +template +class PowGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto factor = static_cast(context.Attr("factor")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + dx.device(place) = dy * factor * x.pow(factor - static_cast(1)); + } +}; + +template +class STanhKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* Y = context.Output("Y"); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + Y->mutable_data(context.GetPlace()); + + auto x = framework::EigenVector::Flatten(*X); + auto y = framework::EigenVector::Flatten(*Y); + auto place = context.GetEigenDevice(); + y.device(place) = scale_b * (scale_a * x).tanh(); + } +}; + +template +class STanhGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* X = context.Input("X"); + auto* dY = context.Input(framework::GradVarName("Y")); + auto* dX = context.Output(framework::GradVarName("X")); + auto scale_a = static_cast(context.Attr("scale_a")); + auto scale_b = static_cast(context.Attr("scale_b")); + dX->mutable_data(context.GetPlace()); + + auto dy = framework::EigenVector::Flatten(*dY); + auto x = framework::EigenVector::Flatten(*X); + auto dx = framework::EigenVector::Flatten(*dX); + auto place = context.GetEigenDevice(); + + auto temp = (scale_a * x).tanh() * (scale_a * x).tanh(); + dx.device(place) = dy * scale_a * scale_b * (static_cast(1) - temp); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/clip_op.cc b/paddle/operators/clip_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..86d79866a8e7c4cda036ce7e0f5527fd0086b482 --- /dev/null +++ b/paddle/operators/clip_op.cc @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_op.h" + +namespace paddle { +namespace operators { + +using framework::LoDTensor; + +class ClipOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of ClipOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) of ClipOp should not be null."); + auto x_dims = ctx.Input("X")->dims(); + auto max = Attr("max"); + auto min = Attr("min"); + PADDLE_ENFORCE_LT(min, max, "max should be greater than min."); + ctx.Output("Out")->Resize(x_dims); + } +}; + +template +class ClipOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ClipOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "(Tensor)The input of clip op." + "The input should be a k-D tensor(k > 0 and k < 7)"); + AddOutput("Out", "(Tensor)The output of clip op with shape as input(X)"); + AddAttr( + "min", "(float)Minimum value, under which element is replaced by min."); + AddAttr( + "max", "(float)Maximum value, above which element is replaced by max"); + AddComment(R"DOC( +Clip operator limits the given input within an interval. The interval is +specified with arguments 'min' and 'max'. +)DOC"); + } +}; + +class ClipOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + if (x_grad != nullptr) { + x_grad->Resize(x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(clip, ops::ClipOp, ops::ClipOpMaker, clip_grad, + ops::ClipOpGrad); +REGISTER_OP_CPU_KERNEL(clip, + ops::ClipKernel); +REGISTER_OP_CPU_KERNEL(clip_grad, + ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.cu b/paddle/operators/clip_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..ca9701298fdae3fabe234925edaf9e4d775cc66e --- /dev/null +++ b/paddle/operators/clip_op.cu @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/clip_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(clip, + ops::ClipKernel); +REGISTER_OP_GPU_KERNEL(clip_grad, + ops::ClipGradKernel); diff --git a/paddle/operators/clip_op.h b/paddle/operators/clip_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ce1d4e1f460414e6e4acee4fa3207f309c55d86b --- /dev/null +++ b/paddle/operators/clip_op.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; +using platform::Transform; + +template +class ClipFunctor { + public: + explicit ClipFunctor(const T min, const T max) : min_(min), max_(max) {} + HOSTDEVICE T operator()(const T& x) const { + if (x < min_) + return min_; + else if (x > max_) + return max_; + else + return x; + } + + private: + T min_; + T max_; +}; + +template +class ClipGradFunctor { + public: + explicit ClipGradFunctor(const T min, const T max) : min_(min), max_(max) {} + HOSTDEVICE T operator()(const T& x, const T& y) const { + return (y > min_ && y < max_) ? x : 0; + } + + private: + T min_; + T max_; +}; + +template +class ClipKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max = context.Attr("max"); + auto min = context.Attr("min"); + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + T* out_data = out->mutable_data(context.GetPlace()); + const T* x_data = x->data(); + int64_t numel = x->numel(); + Transform trans; + trans(context.device_context(), x_data, x_data + numel, out_data, + ClipFunctor(min, max)); + } +}; + +template +class ClipGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto max = context.Attr("max"); + auto min = context.Attr("min"); + auto* d_out = context.Input(framework::GradVarName("Out")); + auto* d_x = context.Output(framework::GradVarName("X")); + if (d_x != nullptr) { + auto* x = context.Input("X"); + int64_t numel = d_out->numel(); + auto* d_x_data = d_x->mutable_data(context.GetPlace()); + const T* d_out_data = d_out->data(); + const T* x_data = x->data(); + Transform trans; + trans(context.device_context(), d_out_data, d_out_data + numel, x_data, + d_x_data, ClipGradFunctor(min, max)); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..12db65b5cbf224e95d91c7b4839afa552c084ee7 --- /dev/null +++ b/paddle/operators/conv2d_op.cc @@ -0,0 +1,133 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/gemm_conv2d_op.h" + +namespace paddle { +namespace operators { + +int outputSize(int input_size, int filter_size, int padding, int stride) { + int output_size = (input_size - filter_size + 2 * padding) / stride + 1; + return output_size; +} + +class Conv2DOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Input"), + "Input(Input) of Conv2DOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Filter"), + "Input(Filter) of Conv2DOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Output"), + "Output(Output) of Conv2DOp should not be null."); + + auto in = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto out = ctx.Output("Output"); + std::vector strides = Attr>("strides"); + std::vector paddings = Attr>("paddings"); + int groups = Attr("groups"); + int input_channels = in->dims()[1]; + int output_channels = filter->dims()[0]; + + PADDLE_ENFORCE_EQ(in->dims().size(), 4, "Conv2DOp input should be 4-D."); + PADDLE_ENFORCE_EQ(filter->dims().size(), 4, + "Conv2DOp filter should be 4-D."); + PADDLE_ENFORCE_EQ(input_channels, filter->dims()[1] * groups, + "The number of input channels should be equal to filter " + "channels * groups."); + PADDLE_ENFORCE_EQ( + output_channels % groups, 0, + "The number of output channels should be divided by groups."); + + auto output_height = + outputSize(in->dims()[2], filter->dims()[2], paddings[0], strides[0]); + auto output_width = + outputSize(in->dims()[3], filter->dims()[3], paddings[1], strides[1]); + out->Resize( + {in->dims()[0], filter->dims()[0], output_height, output_width}); + } +}; + +class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Conv2DOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "The input tensor of convolution operator. " + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of channels, H and W is the height and width of image."); + AddInput( + "Filter", + "The filter tensor of convolution operator." + "The format of the filter tensor is MCHW, where M is the number of " + "output image channels, C is the number of input image channels, " + "H and W is height and width of filter. " + "If the groups attribute is greater than 1, C equal the number of " + "input image channels divided by the groups."); + AddOutput("Output", + "The output tensor of convolution operator." + "The format of output tensor is also NCHW."); + AddAttr>("strides", "strides of convolution operator.") + .SetDefault({1, 1}); + AddAttr>("paddings", "paddings of convolution operator.") + .SetDefault({0, 0}); + AddAttr( + "groups", + "group size of convolution operator. " + "Refer to grouped convolution in Alex Krizhevsky's paper: " + "when group=2, the first half of the filters are only connected to the " + "first half of the input channels, and the second half only connected " + "to the second half.") + .SetDefault(1); + AddComment(R"DOC( +The convolution operation calculates the output based on the input, filter +and strides, paddings, groups parameters. The size of each dimension of the +parameters is checked in the infer-shape. +)DOC"); + } +}; + +class Conv2DOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + auto in = ctx.Input("Input"); + auto filter = ctx.Input("Filter"); + auto d_in = + ctx.Output(framework::GradVarName("Input")); + auto d_filter = + ctx.Output(framework::GradVarName("Filter")); + if (d_in) d_in->Resize(in->dims()); + if (d_filter) d_filter->Resize(filter->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(conv2d, ops::Conv2DOp, ops::Conv2DOpMaker, conv2d_grad, + ops::Conv2DOpGrad); + +REGISTER_OP_CPU_KERNEL( + conv2d, ops::GemmConv2DKernel); +REGISTER_OP_CPU_KERNEL( + conv2d_grad, ops::GemmConvGrad2DKernel); diff --git a/paddle/operators/conv2d_op.cu b/paddle/operators/conv2d_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..5df818ba0496a65502dde37fd1397ec56f8c1101 --- /dev/null +++ b/paddle/operators/conv2d_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/gemm_conv2d_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL( + conv2d, ops::GemmConv2DKernel); +REGISTER_OP_GPU_KERNEL( + conv2d_grad, ops::GemmConvGrad2DKernel); diff --git a/paddle/operators/crop_op.cc b/paddle/operators/crop_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..7ed21f336f69e494f3c4039c609c83407a80cd8c --- /dev/null +++ b/paddle/operators/crop_op.cc @@ -0,0 +1,139 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/crop_op.h" +#include + +namespace paddle { +namespace operators { + +using framework::Tensor; +using framework::LoDTensor; + +class CropOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input(X) of CropOp should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) of CropOp should not be null."); + auto x_dim = ctx.Input("X")->dims(); + auto *y = ctx.Input("Y"); + auto *out = ctx.Output("Out"); + if (y == nullptr) { + auto shape = Attr>("shape"); + PADDLE_ENFORCE_EQ( + int64_t(shape.size()), x_dim.size(), + "Shape size should be equal to dimention size of input tensor."); + std::vector tensor_shape(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + tensor_shape[i] = static_cast(shape[i]); + } + out->Resize(framework::make_ddim(tensor_shape)); + } else { + PADDLE_ENFORCE_EQ(framework::arity(x_dim), framework::arity(y->dims()), + "Tensor rank of both CropOp's " + "inputs must be same."); + out->Resize(y->dims()); + } + } +}; + +class CropOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CropOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input of pad op. " + "The input should be a k-D tensor(k > 0 and k < 7)"); + AddInput("Y", + "The input used as reference for cropping" + " with the same dimension as X. "); + AddOutput("Out", + "The output of crop op " + "with the same dimension as X."); + AddAttr>("offsets", + "A list describing offsets to be cropped." + "The size of offsets list should be as same as " + "dimension size of input X."); + AddAttr>("shape", + "A list describing the shape of output." + "The size of shape list should be as same as " + "dimension size of input X.") + .SetDefault(std::vector()); + AddComment(R"DOC( +Crop Operator. +Crop input into output, as specified by offsets and shape. + +There are two ways to set shape: +1. referenc input: crop input X as shape as reference input. + The dimension of reference input should + be as same as input X. +2. shape list: crop input X by shape described by a list. + The size of shape list should be as same as + dimension size of input X. + +The input should be a k-D tensor(k > 0 and k < 7). As an example: + +Given: + + X = [[0, 1, 2, 0, 0] + [0, 3, 4, 0, 0] + [0, 0, 0, 0, 0]] + +and + + offsets = [0, 1] + +and + + shape = [2, 2] + +then we get + + Out = [[1, 2], + [3, 4]] + +)DOC"); + } +}; + +class CropOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = ctx.Output(framework::GradVarName("X")); + if (x_grad != nullptr) { + x_grad->Resize(x_dims); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(crop, ops::CropOp, ops::CropOpMaker, crop_grad, ops::CropOpGrad); +REGISTER_OP_CPU_KERNEL(crop, ops::CropKernel); +REGISTER_OP_CPU_KERNEL(crop_grad, + ops::CropGradKernel); diff --git a/paddle/operators/crop_op.cu b/paddle/operators/crop_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..f8ee18a1d6e894cbb2d71dd4b6b459abeb076817 --- /dev/null +++ b/paddle/operators/crop_op.cu @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/crop_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(crop, ops::CropKernel); +REGISTER_OP_GPU_KERNEL(crop_grad, + ops::CropGradKernel); diff --git a/paddle/operators/crop_op.h b/paddle/operators/crop_op.h new file mode 100644 index 0000000000000000000000000000000000000000..2f40c059033ec649b29f6ecdee4fcedd128a63a6 --- /dev/null +++ b/paddle/operators/crop_op.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2016 CropdleCropdle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/strided_memcpy.h" + +namespace paddle { +namespace operators { // Internal + +template +using EigenTensor = framework::EigenTensor; +using framework::Tensor; + +template +class CropKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + const T* x_data = x->data(); + T* out_data = out->mutable_data(context.GetPlace()); + auto x_stride = framework::stride(x->dims()); + auto out_stride = framework::stride(out->dims()); + auto offsets = context.Attr>("offsets"); + PADDLE_ENFORCE_EQ( + x->dims().size(), offsets.size(), + "Offsets size should be equal to dimension size of input tensor."); + int64_t offset = 0; + for (int i = 0; i < offsets.size(); ++i) { + offset += (x_stride[i] * offsets[i]); + } + StridedMemcpy(context.device_context(), x_data + offset, x_stride, + out->dims(), out_stride, out_data); + } +}; + +template +void CropGradFunction(const framework::ExecutionContext& context) { + auto* d_x = context.Output(framework::GradVarName("X")); + if (d_x != nullptr) { + auto* d_out = context.Input(framework::GradVarName("Out")); + d_x->mutable_data(context.GetPlace()); + auto offsets = context.Attr>("offsets"); + Eigen::array, D> paddings; + for (int i = 0; i < D; ++i) { + paddings[i].first = offsets[i]; + paddings[i].second = d_x->dims()[i] - d_out->dims()[i] - offsets[i]; + } + auto d_x_tensor = EigenTensor::From(*d_x); + auto d_out_tensor = EigenTensor::From(*d_out); + d_x_tensor.device(context.GetEigenDevice()) = + d_out_tensor.pad(paddings, 0); + } +} + +template +class CropGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + size_t rank = + context.Input(framework::GradVarName("Out"))->dims().size(); + switch (rank) { + case 1: + CropGradFunction(context); + break; + case 2: + CropGradFunction(context); + break; + case 3: + CropGradFunction(context); + break; + case 4: + CropGradFunction(context); + break; + case 5: + CropGradFunction(context); + break; + case 6: + CropGradFunction(context); + break; + default: + PADDLE_THROW( + "CropOp only support tensors with no more than 6 dimensions."); + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..953367eb8bcd1282ab6c7e1189d778f0ce3da541 --- /dev/null +++ b/paddle/operators/cross_entropy_op.cc @@ -0,0 +1,147 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/cross_entropy_op.h" + +namespace paddle { +namespace operators { + +using framework::LoDTensor; + +class CrossEntropyOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), "Output(Y) must not be null."); + + auto x = ctx.Input("X"); + auto label = ctx.Input("Label"); + PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank must be 2."); + PADDLE_ENFORCE_EQ(label->dims().size(), 2, + "Input(Label)'s rank must be 2."); + // TODO(xinghai-sun): remove this check after swtiching to bool + PADDLE_ENFORCE(ctx.Attr("soft_label") == 0 || + ctx.Attr("soft_label") == 1); + PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], + "The 1st dimension of Input(X) and Input(Label) must " + "be equal."); + if (ctx.Attr("soft_label") == 1) { + PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], + "If Attr(soft_label) == 1, The 2nd dimension of " + "Input(X) and Input(Label) must be equal."); + } else { + PADDLE_ENFORCE_EQ(label->dims()[1], 1, + "If Attr(soft_label) == 0, The 2nd dimension of " + "Input(Label) must be 1."); + } + + ctx.Output("Y")->Resize({x->dims()[0], 1}); + } +}; + +class CrossEntropyGradientOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Y")), + "Input(Y@GRAD) must not be null."); + + auto x = ctx.Input("X"); + auto label = ctx.Input("Label"); + auto dy = ctx.Input(framework::GradVarName("Y")); + PADDLE_ENFORCE_EQ(x->dims().size(), 2, "Input(X)'s rank must be 2."); + PADDLE_ENFORCE_EQ(dy->dims().size(), 2, "Input(Y@Grad)'s rank must be 2."); + PADDLE_ENFORCE_EQ(label->dims().size(), 2, + "Input(Label)'s rank must be 2."); + // TODO(xinghai-sun): remove this check after swtiching to bool + PADDLE_ENFORCE(ctx.Attr("soft_label") == 0 || + ctx.Attr("soft_label") == 1); + PADDLE_ENFORCE_EQ(x->dims()[0], label->dims()[0], + "The 1st dimension of Input(X) and Input(Label) must " + "be equal."); + PADDLE_ENFORCE_EQ(x->dims()[0], dy->dims()[0], + "The 1st dimension of Input(X) and Input(Y@Grad) must " + "be equal."); + PADDLE_ENFORCE_EQ(dy->dims()[1], 1, + "The 2nd dimension of Input(Y@Grad) must be 1."); + if (ctx.Attr("soft_label") == 1) { + PADDLE_ENFORCE_EQ(x->dims()[1], label->dims()[1], + "If Attr(soft_label) == 1, The 2nd dimension of " + "Input(X) and Input(Label) must be equal."); + } else { + PADDLE_ENFORCE_EQ(label->dims()[1], 1, + "If Attr(soft_label) == 0, The 2nd dimension of " + "Input(Label) must be 1."); + } + + auto dx = ctx.Output(framework::GradVarName("X")); + dx->Resize(x->dims()); + } +}; + +class CrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { + public: + CrossEntropyOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The first input of CrossEntropyOp"); + AddInput("Label", "The second input of CrossEntropyOp"); + AddOutput("Y", "The output of CrossEntropyOp"); + AddAttr("soft_label", "Is soft label. Default zero.").SetDefault(0); + + AddComment(R"DOC( +CrossEntropy Operator. + +It supports both standard cross-entropy and soft-label cross-entropy loss +computation. +1) One-hot cross-entropy: + soft_label = 0, Label[i, 0] indicates the class index for sample i: + + Y[i] = -log(X[i, Label[i]]) + +2) Soft-label cross-entropy: + soft_label = 1, Label[i, j] indicates the soft label of class j + for sample i: + + Y[i] = \sum_j{-Label[i, j] * log(X[i, j])} + + Please make sure that in this case the summuation of each row of Label + equals one. + +3) One-hot cross-entropy with vecterized Input(Label): + As a special case of 2), when each row of Input(Label) has only one + non-zero element (equals 1), soft-label cross-entropy degenerates to a + one-hot cross-entropy with one-hot label representation. +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(cross_entropy, ops::CrossEntropyOp, ops::CrossEntropyOpMaker, + cross_entropy_grad, ops::CrossEntropyGradientOp); +REGISTER_OP_CPU_KERNEL(cross_entropy, ops::CrossEntropyOpKernel); +REGISTER_OP_CPU_KERNEL(cross_entropy_grad, + ops::CrossEntropyGradientOpKernel); diff --git a/paddle/operators/cross_entropy_op.cu b/paddle/operators/cross_entropy_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..ab6ad0e062269483948bf70e492c9431991221fb --- /dev/null +++ b/paddle/operators/cross_entropy_op.cu @@ -0,0 +1,158 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/operators/cross_entropy_op.h" +#include "paddle/platform/assert.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +template +__global__ void CrossEntropyKernel(T* Y, const T* X, const int* label, + const int N, const int D) { + // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. + // CUDA_1D_KERNEL_LOOP(i, N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + PADDLE_ASSERT(label[i] >= 0 && label[i] < D); + Y[i] = -tolerable_value(log(X[i * D + label[i]])); + } +} + +template +__global__ void SoftCrossEntropyKernel(T* Y, const T* X, const T* label, + const int N, const int D) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + T sum = static_cast(0); + for (int j = 0; j < D; j++) { + sum += label[i * D + j] * tolerable_value(log(X[i * D + j])); + } + Y[i] = -sum; + } +} + +// TODO(qingqing): make zero setting an common function. +template +__global__ void zero(T* X, const int N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + X[i] = 0.0; + } +} + +template +__global__ void CrossEntropyGradientKernel(T* dX, const T* dY, const T* X, + const int* label, const int N, + const int D) { + // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. + // CUDA_1D_KERNEL_LOOP(i, N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + int idx = i * D + label[i]; + dX[idx] = -dY[i] / X[idx]; + } +} + +template +__global__ void SoftCrossEntropyGradientKernel(T* dX, const T* dY, const T* X, + const T* label, const int N, + const int D) { + // TOOD(qingqing): optimize for this kernel + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + for (int j = 0; j < D; ++j) { + int idx = i * D + j; + dX[idx] = -label[idx] * dY[i] / X[idx]; + } + } +} + +template +class CrossEntropyOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + + auto x = ctx.Input("X"); + auto y = ctx.Output("Y"); + auto label = ctx.Input("Label"); + + auto* x_data = x->data(); + y->mutable_data(ctx.GetPlace()); + auto* y_data = y->data(); + + int n = x->dims()[0]; + int d = x->dims()[1]; + int block = 512; + int grid = (n + block - 1) / block; + // TODO(qingqing) launch kernel on specified stream + // base on ExecutionContext. + if (ctx.Attr("soft_label") == 1) { + auto* label_data = ctx.Input("Label")->data(); + SoftCrossEntropyKernel<<>>(y_data, x_data, label_data, n, + d); + } else { + auto* label_data = ctx.Input("Label")->data(); + CrossEntropyKernel<<>>(y_data, x_data, label_data, n, d); + } + } +}; + +template +class CrossEntropyGradientOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "It must use GPUPlace."); + + auto x = ctx.Input("X"); + auto dx = ctx.Output(framework::GradVarName("X")); + auto dy = ctx.Input(framework::GradVarName("Y")); + auto label = ctx.Input("Label"); + + auto* dx_data = dx->mutable_data(ctx.GetPlace()); + auto* dy_data = dy->data(); + auto* x_data = x->data(); + + int n = x->dims()[0]; + int d = x->dims()[1]; + int block = 512; + int grid = (n * d + block - 1) / block; + zero<<>>(dx_data, n * d); + grid = (n + block - 1) / block; + // TODO(qingqing): launch kernel on specified stream + // base on ExecutionContext. + if (ctx.Attr("soft_label") == 1) { + auto* label_data = label->data(); + SoftCrossEntropyGradientKernel<<>>( + dx_data, dy_data, x_data, label_data, n, d); + } else { + auto* label_data = label->data(); + CrossEntropyGradientKernel<<>>(dx_data, dy_data, x_data, + label_data, n, d); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(cross_entropy, ops::CrossEntropyOpCUDAKernel); +REGISTER_OP_GPU_KERNEL(cross_entropy_grad, + ops::CrossEntropyGradientOpCUDAKernel); diff --git a/paddle/operators/cross_entropy_op.h b/paddle/operators/cross_entropy_op.h new file mode 100644 index 0000000000000000000000000000000000000000..1b4b23ac2029138afadef0168262203ac2e20430 --- /dev/null +++ b/paddle/operators/cross_entropy_op.h @@ -0,0 +1,117 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +HOSTDEVICE T tolerable_value(const T x) { + PADDLE_ASSERT(std::is_floating_point::value); + const T kApproInf = 1e20; + if (x == INFINITY) { + return kApproInf; + } + if (x == -INFINITY) { + return -kApproInf; + } + return x; +} + +template +class CrossEntropyOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "It must use CPUPlace."); + + auto x = ctx.Input("X"); + auto y = ctx.Output("Y"); + + auto* x_data = x->data(); + y->mutable_data(ctx.GetPlace()); + auto* y_data = y->data(); + + int batch_size = x->dims()[0]; + int class_num = x->dims()[1]; + + if (ctx.Attr("soft_label") == 1) { + auto* label_data = ctx.Input("Label")->data(); + int index = 0; + for (int i = 0; i < batch_size; ++i) { + T sum = static_cast(0); + for (int j = 0; j < class_num; ++j) { + sum += label_data[index] * tolerable_value(std::log(x_data[index])); + y_data[i] = -sum; + index++; + } + } + } else { + auto* label_data = ctx.Input("Label")->data(); + for (int i = 0; i < batch_size; ++i) { + int index = i * class_num + label_data[i]; + y_data[i] = -tolerable_value(std::log(x_data[index])); + } + } + } +}; + +template +class CrossEntropyGradientOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "It must use CPUPlace."); + + auto x = ctx.Input("X"); + auto dx = ctx.Output(framework::GradVarName("X")); + auto dy = ctx.Input(framework::GradVarName("Y")); + auto label = ctx.Input("Label"); + + auto* dx_data = dx->mutable_data(ctx.GetPlace()); + auto* dy_data = dy->data(); + auto* x_data = x->data(); + + int batch_size = x->dims()[0]; + int class_num = x->dims()[1]; + + // TODO(qingqing): make zero setting an common function. + if (ctx.Attr("soft_label") == 1) { + auto* label_data = ctx.Input("Label")->data(); + int index = 0; + for (int i = 0; i < batch_size; ++i) { + for (int j = 0; j < class_num; ++j) { + dx_data[index] = -label_data[index] * dy_data[i] / x_data[index]; + index++; + } + } + } else { + auto* label_data = label->data(); + memset(dx_data, 0, sizeof(T) * batch_size * class_num); + for (int i = 0; i < batch_size; ++i) { + PADDLE_ASSERT(label_data[i] >= 0 || label_data[i] < class_num); + int index = i * class_num + label_data[i]; + dx_data[index] = -dy_data[i] / x_data[index]; + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/detail/strided_memcpy.h b/paddle/operators/detail/strided_memcpy.h new file mode 100644 index 0000000000000000000000000000000000000000..b165224b37fb091c094a823179256c3dd40a37c9 --- /dev/null +++ b/paddle/operators/detail/strided_memcpy.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/ddim.h" +#include "paddle/memory/memcpy.h" +#include "paddle/platform/device_context.h" + +namespace paddle { +namespace operators { +namespace detail { + +template +struct StridedMemcpyFunctor; + +template +struct StridedMemcpyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim<1> src_stride, framework::Dim<1> dst_dim, + framework::Dim<1> dst_stride, T* dst) const { + auto place = dev_ctx.GetPlace(); + if (platform::is_cpu_place(place)) { + auto& cpu_place = boost::get(place); + memory::Copy(cpu_place, dst, cpu_place, src, sizeof(T) * dst_dim.head); + } else { +#ifndef PADDLE_ONLY_CPU + auto& gpu_place = boost::get(place); + auto& cuda_ctx = + reinterpret_cast(dev_ctx); + memory::Copy(gpu_place, dst, gpu_place, src, sizeof(T) * dst_dim.head, + cuda_ctx.stream()); +#else + PADDLE_THROW("Paddle is not compiled with GPU"); +#endif + } + } +}; + +template +struct StridedMemcpyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim src_stride, framework::Dim dst_dim, + framework::Dim dst_stride, T* dst) const { + for (int64_t i = 0; i < dst_dim.head; ++i) { + StridedMemcpyFunctor func; + func(dev_ctx, src, src_stride.tail, dst_dim.tail, dst_stride.tail, dst); + src += src_stride.head; + dst += dst_stride.head; + } + } +}; + +template +struct StridedCopyDimVisitor : public boost::static_visitor { + StridedCopyDimVisitor(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_stride, T* dst) + : dev_ctx_(dev_ctx), + src_(src), + src_stride_(src_stride), + dst_stride_(dst_stride), + dst_(dst) {} + + template + void operator()(Dim dst_dim) const { + Dim src_stride = boost::get(src_stride_); + Dim dst_stride = boost::get(dst_stride_); + constexpr int dim = Dim::dimensions; + StridedMemcpyFunctor functor; + functor(dev_ctx_, src_, src_stride, dst_dim, dst_stride, dst_); + } + + const platform::DeviceContext& dev_ctx_; + const T* src_; + const framework::DDim& src_stride_; + const framework::DDim& dst_stride_; + T* dst_; +}; + +} // namespace detail +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/dropout_op.cc b/paddle/operators/dropout_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..b111b9fccb2310bd5fb92bda878a497c51f62ce0 --- /dev/null +++ b/paddle/operators/dropout_op.cc @@ -0,0 +1,113 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/dropout_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; +using framework::LoDTensor; + +class DropoutOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); + PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); + // TODO(xinghai-sun): remove this check after swtiching to bool + PADDLE_ENFORCE(ctx.Attr("is_training") == 0 || + ctx.Attr("is_training") == 1); + + auto dims = ctx.Input("X")->dims(); + ctx.Output("Out")->Resize(dims); + if (ctx.Attr("is_training") == 1) { + ctx.Output("Mask")->Resize(dims); + } + } +}; + +template +class DropoutOpMaker : public framework::OpProtoAndCheckerMaker { + public: + DropoutOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddAttr("dropout_prob", "Probability of setting units to zero.") + .SetDefault(.5f); + // TODO(xinghai-sun): use bool for is_training after bool is supported. + AddAttr("is_training", "Whether in training phase.").SetDefault(1); + AddAttr("seed", "Dropout random seed.").SetDefault(0); + AddInput("X", "The input of dropout op."); + AddOutput("Out", "The output of dropout op."); + AddOutput("Mask", "The random sampled dropout mask.").AsIntermediate(); + + AddComment(R"DOC( +Dropout Operator. + +"Dropout" refers to randomly dropping out units in a nerual network. It is a +regularization technique for reducing overfitting by preventing neuron +co-adaption during training. The dropout operator randomly set (according to +the given dropout probability) the outputs of some units to zero, while others +being set to their inputs. +)DOC"); + } +}; + +template +class DropoutOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_EQ(ctx.Attr("is_training"), 1, + "GradOp is only callable when is_training is true"); + + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Mask"), "Mask must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) must not be null."); + + PADDLE_ENFORCE_GE(ctx.Attr("dropout_prob"), 0); + PADDLE_ENFORCE_LE(ctx.Attr("dropout_prob"), 1); + // TODO(xinghai-sun): remove this check after swtiching to bool + PADDLE_ENFORCE(ctx.Attr("is_training") == 0 || + ctx.Attr("is_training") == 1); + auto x_dims = ctx.Input("X")->dims(); + auto out_dims = ctx.Input(framework::GradVarName("Out"))->dims(); + PADDLE_ENFORCE_EQ(x_dims, out_dims, + "Dimensions of Input(X) and Out@Grad must be the same."); + auto mask_dims = ctx.Input("Mask")->dims(); + PADDLE_ENFORCE_EQ(x_dims, mask_dims, + "Dimensions of Input(X) and Mask must be the same."); + + auto *x_grad = ctx.Output(framework::GradVarName("X")); + x_grad->Resize(x_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(dropout, ops::DropoutOp, ops::DropoutOpMaker, dropout_grad, + ops::DropoutOpGrad); +REGISTER_OP_CPU_KERNEL( + dropout, ops::CPUDropoutKernel); +REGISTER_OP_CPU_KERNEL( + dropout_grad, ops::DropoutGradKernel); diff --git a/paddle/operators/dropout_op.cu b/paddle/operators/dropout_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..186237fb238add37f32403309a0f7e8a9846d335 --- /dev/null +++ b/paddle/operators/dropout_op.cu @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include +#include +#include +#include +#include "paddle/operators/dropout_op.h" + +namespace paddle { +namespace operators { + +template +struct MaskGenerator { + AttrType dropout_prob; + int seed; + + __host__ __device__ MaskGenerator(AttrType dropout_prob, int seed) + : dropout_prob(dropout_prob), seed(seed) {} + + __host__ __device__ T operator()(const unsigned int n) const { + thrust::minstd_rand rng; + rng.seed(seed); + thrust::uniform_real_distribution dist(0, 1); + rng.discard(n); + if (dist(rng) < dropout_prob) { + return static_cast(0); + } else { + return static_cast(1); + } + } +}; + +// It seems that Eigen::Tensor::setRandom in GPU will SEGFAULT. +// Use std::random and thrust::random(thrust is a std library in CUDA) to +// implement uniform random. +template +class GPUDropoutKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* y = context.Output("Out"); + y->mutable_data(context.GetPlace()); + AttrType dropout_prob = context.Attr("dropout_prob"); + + auto X = EigenMatrix::Reshape(*x, 1); + auto Y = EigenMatrix::Reshape(*y, 1); + + auto place = context.GetEigenDevice(); + if (context.Attr("is_training") == 1) { + auto* mask = context.Output("Mask"); + auto* mask_data = mask->mutable_data(context.GetPlace()); + int size = framework::product(mask->dims()); + int seed = context.Attr("seed"); + thrust::counting_iterator index_sequence_begin(0); + thrust::transform(index_sequence_begin, index_sequence_begin + size, + thrust::device_ptr(mask_data), + MaskGenerator(dropout_prob, seed)); + auto M = EigenMatrix::Reshape(*mask, 1); + Y.device(place) = X * M; + } else { + Y.device(place) = X * dropout_prob; + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + dropout, ops::GPUDropoutKernel); +REGISTER_OP_GPU_KERNEL( + dropout_grad, ops::DropoutGradKernel); diff --git a/paddle/operators/dropout_op.h b/paddle/operators/dropout_op.h new file mode 100644 index 0000000000000000000000000000000000000000..82eafee0e0e7db7b4b4ae5405f37146d061aefd5 --- /dev/null +++ b/paddle/operators/dropout_op.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; + +template +class CPUDropoutKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* y = context.Output("Out"); + const auto* x_data = x->data(); + auto* y_data = y->mutable_data(context.GetPlace()); + AttrType dropout_prob = context.Attr("dropout_prob"); + + if (context.Attr("is_training") == 1) { + auto* mask = context.Output("Mask"); + auto* mask_data = mask->mutable_data(context.GetPlace()); + int seed = context.Attr("seed"); + std::minstd_rand engine; + engine.seed(seed); + std::uniform_real_distribution dist(0, 1); + size_t size = framework::product(mask->dims()); + for (size_t i = 0; i < size; ++i) { + if (dist(engine) < dropout_prob) { + mask_data[i] = 0; + y_data[i] = 0; + } else { + mask_data[i] = 1; + y_data[i] = x_data[i]; + } + } + } else { + auto X = EigenMatrix::Reshape(*x, 1); + auto Y = EigenMatrix::Reshape(*y, 1); + auto place = context.GetEigenDevice(); + Y.device(place) = X * dropout_prob; + } + } +}; + +template +class DropoutGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + PADDLE_ENFORCE_EQ(context.Attr("is_training"), 1, + "GradOp is only callable when is_training is true"); + + auto* grad_x = context.Output(framework::GradVarName("X")); + auto* grad_y = context.Input(framework::GradVarName("Out")); + auto* mask = context.Input("Mask"); + grad_x->mutable_data(context.GetPlace()); + + auto M = EigenMatrix::Reshape(*mask, 1); + auto dX = EigenMatrix::Reshape(*grad_x, 1); + auto dY = EigenMatrix::Reshape(*grad_y, 1); + + auto place = context.GetEigenDevice(); + dX.device(place) = dY * M; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/gemm_conv2d_op.h b/paddle/operators/gemm_conv2d_op.h new file mode 100644 index 0000000000000000000000000000000000000000..5c9e81732aa72211c2021382cf9a907880c53c17 --- /dev/null +++ b/paddle/operators/gemm_conv2d_op.h @@ -0,0 +1,226 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/im2col.h" +#include "paddle/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class GemmConv2DKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + // The filter will be reshaped in the calculations, + // so here use an assignment operation, + // that avoids modifying the variable in the Scope. + Tensor filter = *context.Input("Filter"); + Tensor* output = context.Output("Output"); + output->mutable_data(context.GetPlace()); + + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + int groups = context.Attr("groups"); + + int batch_size = input->dims()[0]; + int input_channels = input->dims()[1]; + int filter_height = filter.dims()[filter.dims().size() - 2]; + int filter_width = filter.dims()[filter.dims().size() - 1]; + int output_channels = output->dims()[1]; + int output_height = output->dims()[2]; + int output_width = output->dims()[3]; + + paddle::operators::math::Im2ColFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + im2col; + // use col_shape in the im2col calculation + framework::DDim col_shape = {input_channels / groups, filter_height, + filter_width, output_height, output_width}; + // use col_matrix_shape in the gemm calculation + framework::DDim col_matrix_shape = { + input_channels / groups * filter_height * filter_width, + output_height * output_width}; + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix = col; + col_matrix.Resize(col_matrix_shape); + + framework::DDim input_shape = {input->dims()[1], input->dims()[2], + input->dims()[3]}; + framework::DDim filter_matrix_shape = {filter.dims()[0], + filter.numel() / filter.dims()[0]}; + filter.Resize(filter_matrix_shape); + + framework::DDim output_matrix_shape = {output_channels, + output_height * output_width}; + + // convolution operator: im2col + gemm + int in_step = input_channels / groups; + int out_step = output_channels / groups; + for (int i = 0; i < batch_size; i++) { + Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); + Tensor out_batch = output->Slice(i, i + 1).Resize(output_matrix_shape); + for (int g = 0; g < groups; g++) { + // im2col + Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); + im2col(context.device_context(), in_slice, col, strides[0], strides[1], + paddings[0], paddings[1]); + + // gemm + Tensor out_slice = out_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor filter_slice = filter.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), filter_slice, false, + col_matrix, false, T(1.0), &out_slice, T(0.0)); + } + } + } +}; + +template +class GemmConvGrad2DKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + const Tensor* output_grad = + context.Input(framework::GradVarName("Output")); + Tensor* input_grad = + context.Output(framework::GradVarName("Input")); + Tensor* filter_grad = + context.Output(framework::GradVarName("Filter")); + + // The filter and filter_grad will be reshaped in the calculations, + // so here use an assignment operation, + // that avoids modifying the variable in the Scope. + Tensor filter = *context.Input("Filter"); + + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + int groups = context.Attr("groups"); + + int batch_size = input->dims()[0]; + int input_channels = input->dims()[1]; + int filter_height = filter.dims()[filter.dims().size() - 2]; + int filter_width = filter.dims()[filter.dims().size() - 1]; + int output_channels = output_grad->dims()[1]; + int output_height = output_grad->dims()[2]; + int output_width = output_grad->dims()[3]; + + paddle::operators::math::Col2ImFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + col2im; + paddle::operators::math::Im2ColFunctor< + paddle::operators::math::ColFormat::kCFO, Place, T> + im2col; + // use col_shape in the im2col and col2im calculation + framework::DDim col_shape = {input_channels / groups, filter_height, + filter_width, output_height, output_width}; + // use col_matrix_shape in the gemm calculation + framework::DDim col_matrix_shape = { + input_channels / groups * filter_height * filter_width, + output_height * output_width}; + Tensor col; + col.mutable_data(col_shape, context.GetPlace()); + // col_matrix shares the same piece of data with col, + // but will be reshaped into a two-dimensional matrix shape + // to call the matrix multiplication interface. + Tensor col_matrix = col; + col_matrix.Resize(col_matrix_shape); + + framework::DDim input_shape = {input->dims()[1], input->dims()[2], + input->dims()[3]}; + framework::DDim output_matrix_shape = { + output_grad->dims()[1], + output_grad->dims()[2] * output_grad->dims()[3]}; + + framework::DDim filter_matrix_shape = {filter.dims()[0], + filter.numel() / filter.dims()[0]}; + filter.Resize(filter_matrix_shape); + + // convolution backward input operator: gemm + col2im + // convolution backward weight operator: im2col + gemm + int in_step = input_channels / groups; + int out_step = output_channels / groups; + + if (input_grad) { + input_grad->mutable_data(context.GetPlace()); + auto t = framework::EigenVector::Flatten(*input_grad); + t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + + for (int i = 0; i < batch_size; i++) { + Tensor out_grad_batch = + output_grad->Slice(i, i + 1).Resize(output_matrix_shape); + Tensor in_grad_batch = + input_grad->Slice(i, i + 1).Resize(input_shape); + for (int g = 0; g < groups; g++) { + // gemm + Tensor out_grad_slice = + out_grad_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor filter_slice = + filter.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), filter_slice, true, + out_grad_slice, false, T(1.0), &col_matrix, + T(0.0)); + + // col2im + Tensor in_grad_slice = + in_grad_batch.Slice(g * in_step, (g + 1) * in_step); + col2im(context.device_context(), in_grad_slice, col, strides[0], + strides[1], paddings[0], paddings[1]); + } + } + } + + if (filter_grad) { + filter_grad->mutable_data(context.GetPlace()); + Tensor filter_grad_ = *filter_grad; + filter_grad_.Resize(filter_matrix_shape); + auto t = framework::EigenVector::Flatten(filter_grad_); + t.device(context.GetEigenDevice()) = t.constant(static_cast(0)); + + for (int i = 0; i < batch_size; i++) { + Tensor out_grad_batch = + output_grad->Slice(i, i + 1).Resize(output_matrix_shape); + Tensor in_batch = input->Slice(i, i + 1).Resize(input_shape); + for (int g = 0; g < groups; g++) { + // im2col + Tensor out_grad_slice = + out_grad_batch.Slice(g * out_step, (g + 1) * out_step); + Tensor in_slice = in_batch.Slice(g * in_step, (g + 1) * in_step); + im2col(context.device_context(), in_slice, col, strides[0], + strides[1], paddings[0], paddings[1]); + + // gemm + Tensor filter_grad_slice = + filter_grad_.Slice(g * out_step, (g + 1) * out_step); + math::matmul(context.device_context(), out_grad_slice, + false, col_matrix, true, T(1.0), + &filter_grad_slice, T(1.0)); + } + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/im2col.cc b/paddle/operators/math/im2col.cc index 5727c1cab16c1379ffe77f5594c057e93a042785..c08a3380f042886cd400df0d840e61856274619c 100644 --- a/paddle/operators/math/im2col.cc +++ b/paddle/operators/math/im2col.cc @@ -27,9 +27,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -79,9 +80,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -137,9 +138,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -197,9 +199,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; diff --git a/paddle/operators/math/im2col.cu b/paddle/operators/math/im2col.cu index 9bff7bee3c95093852305d392af0949b831e5665..01f60bfe70f844fdcfd5aa481c27d9f12ec51305 100644 --- a/paddle/operators/math/im2col.cu +++ b/paddle/operators/math/im2col.cu @@ -64,9 +64,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -84,9 +85,9 @@ class Im2ColFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + im2col<<(context) + .stream()>>>( im.data(), num_outputs, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width, col.data()); @@ -149,9 +150,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); @@ -174,9 +175,9 @@ class Col2ImFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + col2im<<(context) + .stream()>>>( num_kernels, col.data(), input_height + 2 * padding_height, input_width + 2 * padding_width, input_channels, filter_height, filter_width, stride_height, stride_width, padding_height, @@ -235,9 +236,10 @@ template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -268,9 +270,9 @@ class Im2ColFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + im2colOCF<<(context) + .stream()>>>( im.data(), col.data(), input_channels, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width); @@ -318,9 +320,9 @@ template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context) { + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width) { PADDLE_ENFORCE(im.dims().size() == 3); PADDLE_ENFORCE(col.dims().size() == 5); int input_channels = im.dims()[0]; @@ -351,9 +353,9 @@ class Col2ImFunctor<<< - grid, threads, 0, - reinterpret_cast(context)->stream()>>>( + col2imOCF<<(context) + .stream()>>>( im.data(), col.data(), input_channels, input_height, input_width, filter_height, filter_width, stride_height, stride_width, padding_height, padding_width, output_height, output_width); diff --git a/paddle/operators/math/im2col.h b/paddle/operators/math/im2col.h index 8958c5457cc2c3034c34ca82fb2e98cc06be63c5..7b717e1603c94cd77c74cb0d86f1d23e2692f9d8 100644 --- a/paddle/operators/math/im2col.h +++ b/paddle/operators/math/im2col.h @@ -72,17 +72,18 @@ enum class ColFormat { kCFO = 0, kOCF = 1 }; template class Im2ColFunctor { public: - void operator()(const framework::Tensor& im, framework::Tensor& col, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& im, framework::Tensor& col, int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context); + int padding_width); }; template class Col2ImFunctor { public: - void operator()(framework::Tensor& im, const framework::Tensor& col, - int stride_height, int stride_width, int padding_height, - int padding_width, platform::DeviceContext* context); + void operator()(const platform::DeviceContext& context, framework::Tensor& im, + const framework::Tensor& col, int stride_height, + int stride_width, int padding_height, int padding_width); }; } // namespace math diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc index 4f380388b108dc173d847f027ba5c9db387a87f8..f0b8c885918afe7f80edc465c6d9be7c11ac066f 100644 --- a/paddle/operators/math/im2col_test.cc +++ b/paddle/operators/math/im2col_test.cc @@ -78,8 +78,8 @@ void testIm2col() { PADDLE_THROW("no GPU support"); #endif // PADDLE_ONLY_CPU } - im2col(input, output_cfo, stride, stride, padding, padding, context); - im2col_ocf(input, output_ocf, stride, stride, padding, padding, context); + im2col(*context, input, output_cfo, stride, stride, padding, padding); + im2col_ocf(*context, input, output_ocf, stride, stride, padding, padding); float* out_cfo_ptr; if (paddle::platform::is_cpu_place(*place)) { diff --git a/paddle/operators/math/math_function.cc b/paddle/operators/math/math_function.cc index 1e86fc3d166077265e0f433a6712b0665ea5a152..def4b01da098fc960ce7c0e497732fbcc2579945 100644 --- a/paddle/operators/math/math_function.cc +++ b/paddle/operators/math/math_function.cc @@ -19,12 +19,13 @@ namespace operators { namespace math { template <> -void gemm(const CBLAS_TRANSPOSE transA, +void gemm(const platform::DeviceContext& context, + const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const float alpha, const float* A, - const float* B, const float beta, float* C, - platform::DeviceContext* context) { + const float* B, const float beta, + float* C) { int lda = (transA == CblasNoTrans) ? K : M; int ldb = (transB == CblasNoTrans) ? N : K; int ldc = N; @@ -33,13 +34,13 @@ void gemm(const CBLAS_TRANSPOSE transA, } template <> -void gemm(const CBLAS_TRANSPOSE transA, +void gemm(const platform::DeviceContext& context, + const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const double alpha, const double* A, const double* B, const double beta, - double* C, - platform::DeviceContext* context) { + double* C) { int lda = (transA == CblasNoTrans) ? K : M; int ldb = (transB == CblasNoTrans) ? N : K; int ldc = N; @@ -48,13 +49,10 @@ void gemm(const CBLAS_TRANSPOSE transA, } template <> -void matmul(const framework::Tensor& matrix_a, - bool trans_a, - const framework::Tensor& matrix_b, - bool trans_b, float alpha, - framework::Tensor* matrix_out, - float beta, - platform::DeviceContext* context) { +void matmul( + const platform::DeviceContext& context, const framework::Tensor& matrix_a, + bool trans_a, const framework::Tensor& matrix_b, bool trans_b, float alpha, + framework::Tensor* matrix_out, float beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); auto dim_out = matrix_out->dims(); @@ -74,18 +72,15 @@ void matmul(const framework::Tensor& matrix_a, CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; gemm( - transA, transB, M, N, K, alpha, matrix_a.data(), - matrix_b.data(), beta, matrix_out->data(), context); + context, transA, transB, M, N, K, alpha, matrix_a.data(), + matrix_b.data(), beta, matrix_out->data()); } template <> -void matmul(const framework::Tensor& matrix_a, - bool trans_a, - const framework::Tensor& matrix_b, - bool trans_b, double alpha, - framework::Tensor* matrix_out, - double beta, - platform::DeviceContext* context) { +void matmul( + const platform::DeviceContext& context, const framework::Tensor& matrix_a, + bool trans_a, const framework::Tensor& matrix_b, bool trans_b, double alpha, + framework::Tensor* matrix_out, double beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); auto dim_out = matrix_out->dims(); @@ -105,8 +100,8 @@ void matmul(const framework::Tensor& matrix_a, CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; gemm( - transA, transB, M, N, K, alpha, matrix_a.data(), - matrix_b.data(), beta, matrix_out->data(), context); + context, transA, transB, M, N, K, alpha, matrix_a.data(), + matrix_b.data(), beta, matrix_out->data()); } } // namespace math diff --git a/paddle/operators/math/math_function.cu b/paddle/operators/math/math_function.cu index da40b27c948918e4997f4a046d2145552296158b..71563b77b4b262c3f1e17ae7c4381da56ba780a3 100644 --- a/paddle/operators/math/math_function.cu +++ b/paddle/operators/math/math_function.cu @@ -19,12 +19,13 @@ namespace operators { namespace math { template <> -void gemm(const CBLAS_TRANSPOSE transA, +void gemm(const platform::DeviceContext& context, + const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const float alpha, const float* A, - const float* B, const float beta, float* C, - platform::DeviceContext* context) { + const float* B, const float beta, + float* C) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. int lda = (transA == CblasNoTrans) ? K : M; @@ -35,18 +36,19 @@ void gemm(const CBLAS_TRANSPOSE transA, (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasSgemm( - reinterpret_cast(context)->cublas_handle(), + reinterpret_cast(context) + .cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, N)); } template <> -void gemm(const CBLAS_TRANSPOSE transA, +void gemm(const platform::DeviceContext& context, + const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, const double alpha, const double* A, const double* B, const double beta, - double* C, - platform::DeviceContext* context) { + double* C) { // Note that cublas follows fortran order, so the order is different from // the cblas convention. int lda = (transA == CblasNoTrans) ? K : M; @@ -56,18 +58,16 @@ void gemm(const CBLAS_TRANSPOSE transA, cublasOperation_t cuTransB = (transB == CblasNoTrans) ? CUBLAS_OP_N : CUBLAS_OP_T; PADDLE_ENFORCE(platform::dynload::cublasDgemm( - reinterpret_cast(context)->cublas_handle(), + reinterpret_cast(context) + .cublas_handle(), cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, N)); } template <> -void matmul(const framework::Tensor& matrix_a, - bool trans_a, - const framework::Tensor& matrix_b, - bool trans_b, float alpha, - framework::Tensor* matrix_out, - float beta, - platform::DeviceContext* context) { +void matmul( + const platform::DeviceContext& context, const framework::Tensor& matrix_a, + bool trans_a, const framework::Tensor& matrix_b, bool trans_b, float alpha, + framework::Tensor* matrix_out, float beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); auto dim_out = matrix_out->dims(); @@ -87,18 +87,15 @@ void matmul(const framework::Tensor& matrix_a, CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; gemm( - transA, transB, M, N, K, alpha, matrix_a.data(), - matrix_b.data(), beta, matrix_out->data(), context); + context, transA, transB, M, N, K, alpha, matrix_a.data(), + matrix_b.data(), beta, matrix_out->data()); } template <> -void matmul(const framework::Tensor& matrix_a, - bool trans_a, - const framework::Tensor& matrix_b, - bool trans_b, double alpha, - framework::Tensor* matrix_out, - double beta, - platform::DeviceContext* context) { +void matmul( + const platform::DeviceContext& context, const framework::Tensor& matrix_a, + bool trans_a, const framework::Tensor& matrix_b, bool trans_b, double alpha, + framework::Tensor* matrix_out, double beta) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); auto dim_out = matrix_out->dims(); @@ -118,8 +115,8 @@ void matmul(const framework::Tensor& matrix_a, CBLAS_TRANSPOSE transB = (trans_b == false) ? CblasNoTrans : CblasTrans; gemm( - transA, transB, M, N, K, alpha, matrix_a.data(), - matrix_b.data(), beta, matrix_out->data(), context); + context, transA, transB, M, N, K, alpha, matrix_a.data(), + matrix_b.data(), beta, matrix_out->data()); } } // namespace math diff --git a/paddle/operators/math/math_function.h b/paddle/operators/math/math_function.h index 155589fadb3ed9f59160a750d546dd8093a56cbe..d8518e77fa7b4abdbcf08b7983013c24806e14ca 100644 --- a/paddle/operators/math/math_function.h +++ b/paddle/operators/math/math_function.h @@ -66,16 +66,16 @@ namespace math { // For more detailed info, please refer to // http://www.netlib.org/lapack/explore-html/d4/de2/sgemm_8f.html template -void gemm(const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, - const int M, const int N, const int K, const T alpha, const T* A, - const T* B, const T beta, T* C, platform::DeviceContext* context); +void gemm(const platform::DeviceContext& context, const CBLAS_TRANSPOSE transA, + const CBLAS_TRANSPOSE transB, const int M, const int N, const int K, + const T alpha, const T* A, const T* B, const T beta, T* C); // matrix multiply with continuous memory template -void matmul(const framework::Tensor& matrix_a, bool trans_a, +void matmul(const platform::DeviceContext& context, + const framework::Tensor& matrix_a, bool trans_a, const framework::Tensor& matrix_b, bool trans_b, T alpha, - framework::Tensor* matrix_out, T beta, - platform::DeviceContext* context); + framework::Tensor* matrix_out, T beta); } // namespace math } // namespace operators diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 6c020c4ff7285b43bc5836d80c173d3a068e72b3..7e339457f7f08ff16162f399064a4b4dca594d7f 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -15,8 +15,7 @@ TEST(math_function, notrans_mul_trans) { memcpy(input1_ptr, arr, 6 * sizeof(float)); auto* gpu_place = new paddle::platform::GPUPlace(0); - paddle::platform::DeviceContext* context = - new paddle::platform::CUDADeviceContext(*gpu_place); + paddle::platform::CUDADeviceContext context(*gpu_place); input1_gpu.CopyFrom(input1, *gpu_place); input2_gpu.CopyFrom(input1, *gpu_place); @@ -24,7 +23,7 @@ TEST(math_function, notrans_mul_trans) { out_gpu.mutable_data({2, 2}, *gpu_place); paddle::operators::math::matmul( - input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0, context); + context, input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0); out.CopyFrom(out_gpu, *cpu_place); @@ -33,6 +32,7 @@ TEST(math_function, notrans_mul_trans) { EXPECT_EQ(out_ptr[1], 14); EXPECT_EQ(out_ptr[2], 14); EXPECT_EQ(out_ptr[3], 50); + delete gpu_place; } TEST(math_function, trans_mul_notrans) { @@ -48,8 +48,7 @@ TEST(math_function, trans_mul_notrans) { memcpy(input1_ptr, arr, 6 * sizeof(float)); auto* gpu_place = new paddle::platform::GPUPlace(0); - paddle::platform::DeviceContext* context = - new paddle::platform::CUDADeviceContext(*gpu_place); + paddle::platform::CUDADeviceContext context(*gpu_place); input1_gpu.CopyFrom(input1, *gpu_place); input2_gpu.CopyFrom(input1, *gpu_place); @@ -57,7 +56,7 @@ TEST(math_function, trans_mul_notrans) { out_gpu.mutable_data({3, 3}, *gpu_place); paddle::operators::math::matmul( - input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0, context); + context, input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0); out.CopyFrom(out_gpu, *cpu_place); @@ -71,5 +70,6 @@ TEST(math_function, trans_mul_notrans) { EXPECT_EQ(out_ptr[6], 15); EXPECT_EQ(out_ptr[7], 22); EXPECT_EQ(out_ptr[8], 29); + delete gpu_place; } #endif diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..6fe018f9a8fd74479a2feed07379c3179b7c72bd --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cc @@ -0,0 +1,114 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/modified_huber_loss_op.h" + +namespace paddle { +namespace operators { + +class ModifiedHuberLossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + PADDLE_ENFORCE_NOT_NULL(context.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(context.InputVar("Y"), "Y must be initialized."); + + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "The shape of X and Y must be the same."); + PADDLE_ENFORCE_EQ(x->dims().size(), 2, "The tensor rank of X must be 2."); + PADDLE_ENFORCE_EQ(x->dims()[1], 1, "The 2nd dimension of X must be 1."); + + context.Output("IntermediateVal")->Resize(x->dims()); + context.Output("Out")->Resize({x->dims()[0], 1}); + } +}; + +class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ModifiedHuberLossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input tensor of modified huber loss op." + "X is 2-D tensor with shape [batch_size, 1]."); + AddInput("Y", + "The target labels of modified huber loss op." + "The shape of Y is same as X. Values of Y must be 0 or 1."); + AddOutput("IntermediateVal", + "Variable to save intermediate result which will be reused in " + "backward processing.") + .AsIntermediate(); + AddOutput("Out", "Classification loss for X."); + AddComment(R"DOC( +Modified huber loss is used in binary classification problem. The shape of +input X and target Y are both [N, 1] and so is the shape of output loss. +Since target Y is not differentiable, cacluating gradient for Y is illegal. +The formulation of modified huber loss is: + +L(y, f(x)) = max(0, 1 - yf(x))^2 for yf(x) >= -1, + -4yf(x) otherwise. + +Make sure the values of target label Y are in {0, 1} here. The operator will +scale values of Y to {-1, +1} when computing losses and gradients. +)DOC"); + } +}; + +class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + auto* intermediate_val = context.Input("IntermediateVal"); + auto* out_grad = context.Input(framework::GradVarName("Out")); + auto* x_grad = + context.Output(framework::GradVarName("X")); + + PADDLE_ENFORCE_NOT_NULL(x, "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(y, "Y must be initialized."); + PADDLE_ENFORCE_NOT_NULL(intermediate_val, + "Intermediate value must not be null."); + PADDLE_ENFORCE_NOT_NULL(out_grad, "Input(Out@Grad) must not be null."); + + PADDLE_ENFORCE_EQ( + intermediate_val->dims(), x->dims(), + "The shape of X and intermediate value must be the same."); + PADDLE_ENFORCE_EQ(out_grad->dims(), x->dims(), + "The shape of Input(Out@Grad) and X must be the same."); + + if (x_grad) x_grad->Resize(x->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(modified_huber_loss, ops::ModifiedHuberLossOp, + ops::ModifiedHuberLossOpMaker, modified_huber_loss_grad, + ops::ModifiedHuberLossGradOp); + +REGISTER_OP_CPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_CPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradCPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.cu b/paddle/operators/modified_huber_loss_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..bce760f95e72cfec05b07591e0fa1250168b112f --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cu @@ -0,0 +1,78 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include +#include +#include +#include +#include "paddle/framework/op_registry.h" +#include "paddle/operators/modified_huber_loss_op.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +struct ModifiedHuberLossBackward { + template + HOSTDEVICE void operator()(Tuple t) const { + auto inter_val = thrust::get<1>(t); + auto y_val = thrust::get<2>(t); + auto out_grad = thrust::get<3>(t); + if (inter_val < -1) { + thrust::get<0>(t) = -4 * (2 * y_val - 1) * out_grad; + } else if (inter_val < 1) { + thrust::get<0>(t) = -2 * (1 - inter_val) * (2 * y_val - 1) * out_grad; + } else { + thrust::get<0>(t) = 0; + } + } +}; + +template +class ModifiedHuberLossGradGPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("IntermediateVal"); + auto* in2 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + + if (out0) { + auto counts = framework::product(in1->dims()); + auto y_ptr = thrust::device_pointer_cast(in0->data()); + auto inter_val_ptr = thrust::device_pointer_cast(in1->data()); + auto out_grad_ptr = thrust::device_pointer_cast(in2->data()); + thrust::device_ptr x_grad_ptr( + out0->mutable_data(context.GetPlace())); + + auto iter_begin = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr, inter_val_ptr, y_ptr, out_grad_ptr)); + + auto iter_end = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr + counts, inter_val_ptr + counts, + y_ptr + counts, out_grad_ptr + counts)); + + thrust::for_each(iter_begin, iter_end, ModifiedHuberLossBackward()); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_GPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradGPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..2b2aae17084992c4935a697763ff902e455dfcbd --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.h @@ -0,0 +1,107 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +struct CheckLabelValue { + HOSTDEVICE T operator()(const T& val) const { + PADDLE_ASSERT(val == static_cast(0) || val == static_cast(1)); + } +}; + +template +struct ModifiedHuberLossForward { + HOSTDEVICE T operator()(const T& val) const { + if (val < -1) { + return -4 * val; + } else if (val < 1) { + return (1 - val) * (1 - val); + } else { + return static_cast(0); + } + } +}; + +template +class ModifiedHuberLossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* out0 = context.Output("IntermediateVal"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + // make sure value's of Y in {0, 1} + y.unaryExpr(CheckLabelValue()); + auto inter_val = EigenVector::Flatten(*out0); + // scale y to {-1, +1} and compute x * y + inter_val.device(place) = x * (2 * y - static_cast(1)); + auto loss = EigenVector::Flatten(*out1); + loss.device(place) = inter_val.unaryExpr(ModifiedHuberLossForward()); + } +}; + +// CPU backward kernel +template +class ModifiedHuberLossGradCPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("IntermediateVal"); + auto* in2 = + context.Input(framework::GradVarName("Out")); + auto* out0 = + context.Output(framework::GradVarName("X")); + + if (out0) { + const T* y_ptr = in0->data(); + const T* inter_val_ptr = in1->data(); + const T* out_grad_ptr = in2->data(); + size_t counts = static_cast(framework::product(in1->dims())); + T* x_grad_ptr = out0->mutable_data(context.GetPlace()); + for (size_t i = 0; i < counts; ++i) { + if (inter_val_ptr[i] < -1) { + x_grad_ptr[i] = -4 * (2 * y_ptr[i] - 1) * out_grad_ptr[i]; + } else if (inter_val_ptr[i] < 1) { + x_grad_ptr[i] = -2 * (1 - inter_val_ptr[i]) * (2 * y_ptr[i] - 1) * + out_grad_ptr[i]; + } else { + x_grad_ptr[i] = 0; + } + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/mul_op.h b/paddle/operators/mul_op.h index 3c01f868bda8cba488b3403df456d63d6b082fa6..ac7136a76933d1f3ead86518c65d589747227631 100644 --- a/paddle/operators/mul_op.h +++ b/paddle/operators/mul_op.h @@ -46,10 +46,8 @@ class MulKernel : public framework::OpKernel { : *y; z->mutable_data(context.GetPlace()); - auto* device_context = - const_cast(context.device_context_); - math::matmul(x_matrix, false, y_matrix, false, 1, z, 0, - device_context); + math::matmul(context.device_context(), x_matrix, false, y_matrix, + false, 1, z, 0); } }; @@ -71,16 +69,14 @@ class MulGradKernel : public framework::OpKernel { Tensor* dx = ctx.Output(framework::GradVarName("X")); Tensor* dy = ctx.Output(framework::GradVarName("Y")); - auto* device_context = - const_cast(ctx.device_context_); if (dx) { dx->mutable_data(ctx.GetPlace()); Tensor dx_matrix = dx->dims().size() > 2 ? framework::ReshapeToMatrix( *dx, x_num_col_dims) : *dx; // dx = dout * y'. dx: M x K, dout : M x N, y : K x N - math::matmul(*dout, false, y_matrix, true, 1, &dx_matrix, 0, - device_context); + math::matmul(ctx.device_context(), *dout, false, y_matrix, true, + 1, &dx_matrix, 0); } if (dy) { dy->mutable_data(ctx.GetPlace()); @@ -88,8 +84,8 @@ class MulGradKernel : public framework::OpKernel { *dy, y_num_col_dims) : *dy; // dy = x' * dout. dy K x N, dout : M x N, x : M x K - math::matmul(x_matrix, true, *dout, false, 1, &dy_matrix, 0, - device_context); + math::matmul(ctx.device_context(), x_matrix, true, *dout, false, + 1, &dy_matrix, 0); } } }; diff --git a/paddle/operators/onehot_cross_entropy_op.cc b/paddle/operators/onehot_cross_entropy_op.cc deleted file mode 100644 index f38be3549f3c5d2443f61739fc32cdca74197649..0000000000000000000000000000000000000000 --- a/paddle/operators/onehot_cross_entropy_op.cc +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/operators/onehot_cross_entropy_op.h" - -namespace paddle { -namespace operators { - -class OnehotCrossEntropyOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("X"), - "Input(X) of OnehotCrossEntropyOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.InputVar("label"), - "Input(label) of OnehotCrossEntropyOp should not be null."); - PADDLE_ENFORCE_NOT_NULL( - ctx.OutputVar("Y"), - "Output(Y) of OnehotCrossEntropyOp should not be null."); - - auto *X = ctx.Input("X"); - auto *label = ctx.Input("label"); - - PADDLE_ENFORCE_EQ(X->dims().size(), 2, "X's dimension must be 2."); - PADDLE_ENFORCE_EQ(label->dims().size(), 1, "label's dimension must be 1."); - PADDLE_ENFORCE_EQ(X->dims()[0], label->dims()[0]); - ctx.Output("Y")->Resize({X->dims()[0], 1}); - } -}; - -class OnehotCrossEntropyGradientOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - auto dX = ctx.Output(framework::GradVarName("X")); - auto X = ctx.Input("X"); - - dX->Resize(X->dims()); - } -}; - -class OnehotCrossEntropyOpMaker : public framework::OpProtoAndCheckerMaker { - public: - OnehotCrossEntropyOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of OnehotCrossEntropyOp"); - AddInput("label", "The second input of OnehotCrossEntropyOp"); - AddOutput("Y", "The output of OnehotCrossEntropyOp"); - AddComment(R"DOC( -OnehotCrossEntropy Operator. - - Y[i] = -log(X[i][j]) - -)DOC"); - } -}; -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(onehot_cross_entropy, ops::OnehotCrossEntropyOp, - ops::OnehotCrossEntropyOpMaker, onehot_cross_entropy_grad, - ops::OnehotCrossEntropyGradientOp); -REGISTER_OP_CPU_KERNEL(onehot_cross_entropy, - ops::OnehotCrossEntropyOpKernel); -REGISTER_OP_CPU_KERNEL(onehot_cross_entropy_grad, - ops::OnehotCrossEntropyGradientOpKernel); diff --git a/paddle/operators/onehot_cross_entropy_op.cu b/paddle/operators/onehot_cross_entropy_op.cu deleted file mode 100644 index d999bfce58c8a6db5c811aad677c07094b881841..0000000000000000000000000000000000000000 --- a/paddle/operators/onehot_cross_entropy_op.cu +++ /dev/null @@ -1,133 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/framework/op_registry.h" -#include "paddle/platform/assert.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; - -template -__host__ __device__ T clipping_log(const T x) { - PADDLE_ASSERT(std::is_floating_point::value); - const T kApproInf = 1e20; - T v = log(x); - if (v == INFINITY) { - return kApproInf; - } - if (v == -INFINITY) { - return -kApproInf; - } - return v; -} - -template -__global__ void CrossEntropyKernel(T* Y, const T* X, const int* label, - const int N, const int D) { - // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. - // CUDA_1D_KERNEL_LOOP(i, N) { - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; - i += blockDim.x * gridDim.x) { - PADDLE_ASSERT(label[i] >= 0 && label[i] < D); - Y[i] = -clipping_log(X[i * D + label[i]]); - } -} - -// TODO(qingqing): make zero setting an common function. -template -__global__ void zero(T* X, const int N) { - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; - i += blockDim.x * gridDim.x) { - X[i] = 0.0; - } -} - -template -__global__ void CrossEntropyGradientKernel(T* dX, const T* dY, const T* X, - const int* label, const int N, - const int D) { - // TOOD(qingqing) define CUDA_1D_KERNEL_LOOP macro in a common file. - // CUDA_1D_KERNEL_LOOP(i, N) { - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; - i += blockDim.x * gridDim.x) { - int idx = i * D + label[i]; - dX[idx] = -dY[i] / X[idx]; - } -} - -template -class OnehotCrossEntropyOpCUDAKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), - "It must use GPUPlace."); - - auto X = ctx.Input("X"); - const T* Xdata = X->data(); - const int* label_data = ctx.Input("label")->data(); - auto Y = ctx.Output("Y"); - Y->mutable_data(ctx.GetPlace()); - T* Ydata = Y->data(); - - int N = X->dims()[0]; - int D = X->dims()[1]; - int block = 512; - int grid = (N + block - 1) / block; - // TODO(qingqing) launch kernel on specified stream - // base on ExecutionContext. - CrossEntropyKernel<<>>(Ydata, Xdata, label_data, N, D); - } -}; - -template -class OnehotCrossEntropyGradientOpCUDAKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), - "It must use GPUPlace."); - - auto X = ctx.Input("X"); - auto dX = ctx.Output(framework::GradVarName("X")); - auto dY = ctx.Input(framework::GradVarName("Y")); - auto label = ctx.Input("label"); - - auto* dXdata = dX->template mutable_data(ctx.GetPlace()); - auto* dYdata = dY->template data(); - auto* Xdata = X->template data(); - auto* label_data = label->data(); - - int N = X->dims()[0]; - int D = X->dims()[1]; - int block = 512; - int grid = (N * D + block - 1) / block; - zero<<>>(dXdata, N * D); - - grid = (N + block - 1) / block; - // TODO(qingqing): launch kernel on specified stream - // base on ExecutionContext. - CrossEntropyGradientKernel<<>>(dXdata, dYdata, Xdata, - label_data, N, D); - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(onehot_cross_entropy, - ops::OnehotCrossEntropyOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(onehot_cross_entropy_grad, - ops::OnehotCrossEntropyGradientOpCUDAKernel); diff --git a/paddle/operators/onehot_cross_entropy_op.h b/paddle/operators/onehot_cross_entropy_op.h deleted file mode 100644 index eb4d1348de1d940e2648c83c8ba94b289f10c5b2..0000000000000000000000000000000000000000 --- a/paddle/operators/onehot_cross_entropy_op.h +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; - -template -inline T tolerable_value(const T x) { - static_assert(std::is_floating_point::value, - "tolerable_value works only on float, " - "double and double double."); - - const T kApproInf = 1e20; - - if (x == INFINITY) { - return kApproInf; - } - - if (x == -INFINITY) { - return -kApproInf; - } - - return x; -} - -template -class OnehotCrossEntropyOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), - "It must use CPUPlace."); - - auto X = ctx.Input("X"); - const T* Xdata = X->data(); - const int* label_data = ctx.Input("label")->data(); - auto Y = ctx.Output("Y"); - - Y->mutable_data(ctx.GetPlace()); - - T* Ydata = Y->data(); - - int batch_size = X->dims()[0]; - int class_num = X->dims()[1]; - - for (int i = 0; i < batch_size; ++i) { - int index = i * class_num + label_data[i]; - Ydata[i] = -tolerable_value(std::log(Xdata[index])); - } - } -}; - -template -class OnehotCrossEntropyGradientOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), - "It must use CPUPlace."); - - auto X = ctx.Input("X"); - auto dX = ctx.Output(framework::GradVarName("X")); - auto dY = ctx.Input(framework::GradVarName("Y")); - auto label = ctx.Input("label"); - - auto* dXdata = dX->template mutable_data(ctx.GetPlace()); - auto* dYdata = dY->template data(); - auto* Xdata = X->template data(); - auto* label_data = label->data(); - - const int batch_size = X->dims()[0]; - const int class_num = X->dims()[1]; - - // TODO(qingqing): make zero setting an common function. - memset(dXdata, 0, sizeof(T) * batch_size * class_num); - for (int i = 0; i < batch_size; ++i) { - int index = i * class_num + label_data[i]; - dXdata[index] = -tolerable_value(dYdata[i] / Xdata[index]); - } - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/prelu_op.cc b/paddle/operators/prelu_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..7ae80b296850f2f433c89d904ebf32355b2a29c7 --- /dev/null +++ b/paddle/operators/prelu_op.cc @@ -0,0 +1,94 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/prelu_op.h" +#include "paddle/operators/net_op.h" + +namespace paddle { +namespace operators { + +class PReluOp : public framework::OperatorWithKernel { + public: + PReluOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + auto *in = ctx.Input("X"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Alpha"), + "Input(Alpha) should not be null"); + auto *alpha = ctx.Input("Alpha"); + PADDLE_ENFORCE(alpha->numel() == 1, "Size of weight Alpha must be one."); + + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) should not be null"); + auto *out = ctx.Output("Out"); + out->Resize(in->dims()); + } +}; + +class PReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + PReluOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The input tensor of prelu operator."); + AddInput("Alpha", "The alpha weight of PRelu operator."); + AddOutput("Out", "The output tensor of PRelu operator."); + AddComment(R"DOC(PRelu operator + +The equation is: + + f(x) = alpha * x , for x < 0 + f(x) = x , for x >= 0 + +)DOC"); + } +}; + +// The operator to calculate gradients of a prelu operator. +class PReluGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) must not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto *dx = ctx.Output(framework::GradVarName("X")); + auto *x = ctx.Input("X"); + + auto *dalpha = + ctx.Output(framework::GradVarName("Alpha")); + auto *alpha = ctx.Input("Alpha"); + + dx->Resize(x->dims()); + dalpha->Resize(alpha->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP(prelu, ops::PReluOp, ops::PReluOpMaker, prelu_grad, + ops::PReluGradOp); +REGISTER_OP_CPU_KERNEL(prelu, + ops::PReluKernel); +REGISTER_OP_CPU_KERNEL(prelu_grad, + ops::PReluGradKernel); diff --git a/paddle/operators/prelu_op.cu b/paddle/operators/prelu_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..9e391dabae735cc8a740b46b50d31d271f99b65d --- /dev/null +++ b/paddle/operators/prelu_op.cu @@ -0,0 +1,21 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/prelu_op.h" + +REGISTER_OP_GPU_KERNEL( + prelu, paddle::operators::PReluKernel); +REGISTER_OP_GPU_KERNEL( + prelu_grad, + paddle::operators::PReluGradKernel); diff --git a/paddle/operators/prelu_op.h b/paddle/operators/prelu_op.h new file mode 100644 index 0000000000000000000000000000000000000000..6b78ed295cbac060d816fb3dd27a4b80145cb1ce --- /dev/null +++ b/paddle/operators/prelu_op.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/transform.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +using platform::Transform; + +template +class PReluFunctor { + public: + explicit PReluFunctor(const T* alpha) : alpha_(alpha) {} + + HOSTDEVICE T operator()(const T& x) const { + if (x > 0) + return x; + else + return x * (*alpha_); + } + + private: + const T* alpha_; +}; + +template +class PReluKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* alpha = context.Input("Alpha"); + auto* out = context.Output("Out"); + + const T* x_ptr = x->data(); + T* o_ptr = out->mutable_data(context.GetPlace()); + + auto* alpha_ptr = alpha->data(); + + int numel = x->numel(); + + Transform trans; + trans(context.device_context(), x_ptr, x_ptr + numel, o_ptr, + PReluFunctor(alpha_ptr)); + } +}; + +template +class PReluGradFunctor { + public: + explicit PReluGradFunctor(const T* alpha) : alpha_(alpha) {} + + HOSTDEVICE T operator()(const T& out, const T& dout) const { + if (out > 0) + return dout; + else + return dout * (*alpha_); + } + + private: + const T* alpha_; +}; + +template +class PReluGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* dx = context.Output(framework::GradVarName("X")); + auto* dout = context.Input(framework::GradVarName("Out")); + + auto* out = context.Input("Out"); + auto* alpha = context.Input("Alpha"); + auto* alpha_ptr = alpha->data(); + + T* dx_ptr = dx->mutable_data(context.GetPlace()); + const T* dout_ptr = dout->data(); + const T* out_ptr = out->data(); + int numel = dx->numel(); + + Transform trans; + trans(context.device_context(), out_ptr, out_ptr + numel, dout_ptr, dx_ptr, + PReluGradFunctor(alpha_ptr)); + + // TODO(Zhuoyuan): add dalpha upgrade when GPU kernels ready + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/rank_loss_op.cc b/paddle/operators/rank_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..4bba4200728ebf7e7810ed935f6fdf51c96cbc7a --- /dev/null +++ b/paddle/operators/rank_loss_op.cc @@ -0,0 +1,126 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/rank_loss_op.h" + +namespace paddle { +namespace operators { + +class RankLossOp : public framework::OperatorWithKernel { + public: + RankLossOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + // input check + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) shouldn't be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), + "Input(Left) shouldn't be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), + "Input(Right) shouldn't be null"); + auto label_dims = ctx.Input("Label")->dims(); + auto left_dims = ctx.Input("Left")->dims(); + auto right_dims = ctx.Input("Right")->dims(); + PADDLE_ENFORCE((label_dims == left_dims) && (left_dims == right_dims), + "All inputs must have the same size"); + PADDLE_ENFORCE((label_dims.size() == 2) && (label_dims[1] == 1), + "All inputs must be row vector with size batch_size x 1."); + ctx.Output("Out")->Resize(label_dims); + } +}; + +class RankLossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + RankLossOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Label", + "The label indicating A ranked higher than B or not, row vector."); + AddInput("Left", "The output of RankNet for doc A, vector."); + AddInput("Right", "The output of RankNet for doc B, vetor"); + AddOutput("Out", "The output loss of RankLoss operator, vector."); + AddComment(R"DOC(RankLoss operator + +Rank loss operator for RankNet[1]. RankNet is a pairwise ranking model with +one training sample consisting of a pair of doc A and B, and the label P +indicating that A is ranked higher than B or not: + +P = {0, 1} or {0, 0.5, 1}, where 0.5 means no information about the rank of +the input pair. + +The RankLoss operator contains three inputs: Left (o_i), Right (o_j) and Label +(P_{i,j}), which represent the output of RankNet for two docs and the label +respectively, and yields the rank loss C_{i,j} by following the expression + +\f[ + C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + log(1 + e^{o_{i,j}}) \\ + o_{i,j} = o_i - o_j \\ + \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} +\f] + +The operator can take inputs of one sample or in batch. + +[1]. Chris Burges, Tal Shaked, Erin Renshaw, et al. Learning to + Rank using Gradient Descent. + http://icml.cc/2015/wp-content/uploads/2015/06/icml_ranking.pdf +)DOC"); + } +}; + +class RankLossGradOp : public framework::OperatorWithKernel { + public: + RankLossGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Label"), + "Input(Label) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Left"), + "Input(Left) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Right"), + "Input(Right) shouldn't be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) shouldn't be null."); + auto dims = ctx.Input("Left")->dims(); + auto *left_grad = + ctx.Output(framework::GradVarName("Left")); + auto *right_grad = + ctx.Output(framework::GradVarName("Right")); + if (left_grad) { + left_grad->Resize(dims); + } + if (right_grad) { + right_grad->Resize(dims); + } + } +}; + +} // namespace operators +} // namespace paddle +namespace ops = paddle::operators; + +REGISTER_OP(rank_loss, ops::RankLossOp, ops::RankLossOpMaker, rank_loss_grad, + ops::RankLossGradOp); +REGISTER_OP_CPU_KERNEL(rank_loss, + ops::RankLossKernel); +REGISTER_OP_CPU_KERNEL( + rank_loss_grad, ops::RankLossGradKernel); diff --git a/paddle/operators/rank_loss_op.cu b/paddle/operators/rank_loss_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..779588ff36c792b8925a535d60f1cfbbe3c66d86 --- /dev/null +++ b/paddle/operators/rank_loss_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/rank_loss_op.h" + +REGISTER_OP_GPU_KERNEL( + rank_loss, + paddle::operators::RankLossKernel); +REGISTER_OP_GPU_KERNEL( + rank_loss_grad, + paddle::operators::RankLossGradKernel); diff --git a/paddle/operators/rank_loss_op.h b/paddle/operators/rank_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..9776d123fe4b0cb0cd16a15770fcf42a966fa011 --- /dev/null +++ b/paddle/operators/rank_loss_op.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class RankLossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* out_t = ctx.Output("Out"); + auto* label_t = ctx.Input("Label"); + auto* left_t = ctx.Input("Left"); + auto* right_t = ctx.Input("Right"); + out_t->mutable_data(ctx.GetPlace()); + + auto out = framework::EigenVector::Flatten(*out_t); + auto label = framework::EigenVector::Flatten(*label_t); + auto left = framework::EigenVector::Flatten(*left_t); + auto right = framework::EigenVector::Flatten(*right_t); + + auto& dev = ctx.GetEigenDevice(); + out.device(dev) = + (1. + (left - right).exp()).log() - label * (left - right); + } +}; + +template +class RankLossGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto* d_left_t = + ctx.Output(framework::GradVarName("Left")); + auto* d_right_t = + ctx.Output(framework::GradVarName("Right")); + + auto* d_out_t = ctx.Input(framework::GradVarName("Out")); + auto* label_t = ctx.Input("Label"); + auto* left_t = ctx.Input("Left"); + auto* right_t = ctx.Input("Right"); + + auto& dev = ctx.GetEigenDevice(); + auto d_out = framework::EigenVector::Flatten(*d_out_t); + auto label = framework::EigenVector::Flatten(*label_t); + auto left = framework::EigenVector::Flatten(*left_t); + auto right = framework::EigenVector::Flatten(*right_t); + + // compute d_left + if (d_left_t) { + d_left_t->mutable_data(ctx.GetPlace()); + auto d_left = framework::EigenVector::Flatten(*d_left_t); + d_left.device(dev) = d_out * (1. / (1. + (right - left).exp()) - label); + } + // compute d_right + if (d_right_t) { + d_right_t->mutable_data(ctx.GetPlace()); + auto d_right = framework::EigenVector::Flatten(*d_right_t); + d_right.device(dev) = + -d_out * (1.0 / (1. + (right - left).exp()) - label); + } + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index d3413d7cb9305732e9ddf3cb1bc267f7203097f3..ad985839f5908d9235a4dbefc9b841362810114e 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -29,9 +29,11 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; void RecurrentAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = scope.FindVar((arg_->inlinks[0]).external) - ->GetMutable() - ->dims()[0]; + auto* input0 = scope.FindVar(arg_->inlinks[0]); + PADDLE_ENFORCE_NOT_NULL(input0); + seq_len_ = input0->GetMutable()->dims()[0]; + PADDLE_ENFORCE_GT(seq_len_, 0); + CreateScopes(scope); auto step_scopes = GetStepScopes(scope); rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, @@ -123,14 +125,12 @@ void RecurrentAlgorithm::InitMemories(Scope* step_scope, } const rnn::ArgumentName RecurrentOp::kArgName{ - "step_net", "step_scopes", "inlinks", - "outlinks", "inlink_alias", "outlink_alias", + "step_net", "step_scopes", "inlinks", "outlinks", "memories", "pre_memories", "boot_memories"}; const rnn::ArgumentName RecurrentGradientOp::kArgName{ - "step_net", "step_scopes", "outlink@grad", - "inlink@grad", "inlink_alias", "outlink_alias", - "memories", "pre_memories", "boot_memories@grad"}; + "step_net", "step_scopes", "outlink@grad", "inlink@grad", + "memories", "pre_memories", "boot_memories@grad"}; RecurrentOp::RecurrentOp(const std::string& type, const framework::VariableNameMap& inputs, @@ -160,8 +160,6 @@ class RecurrentAlgorithmProtoAndCheckerMaker AddOutput(name.step_scopes, "step scopes"); // Attributes stored in AttributeMap - AddAttr>(name.inlink_alias, "alias of inlinks"); - AddAttr>(name.outlink_alias, "alias of outlinks"); AddAttr>(name.pre_memories, "names of pre-memories"); AddAttr>(name.memories, "names of memories"); @@ -206,9 +204,8 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients( } void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = scope.FindVar((arg_->inlinks[0]).external) - ->GetMutable() - ->dims()[0]; + seq_len_ = + scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; auto step_scopes = GetStepScopes(scope); rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, true /*infer_shape_mode*/); diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index 6c082cb1825e04accb09019fef28eb2ec6523a5b..ca7219b26d83eb6b8db75a5ed9cd360c5ac1d5df 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -24,22 +24,23 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; void SegmentInputs(const std::vector& step_scopes, - const std::vector& inlinks, const size_t seq_len, - bool infer_shape_mode) { + const std::vector& inlinks, + const size_t seq_len, bool infer_shape_mode) { PADDLE_ENFORCE(!inlinks.empty(), "no in links are provided."); for (size_t i = 0; i < inlinks.size(); ++i) { - auto input_var = step_scopes[0]->FindVar(inlinks[i].external); - PADDLE_ENFORCE(input_var != nullptr, "input link [%s] is not in scope.", - inlinks[i].external); + // global inputs + auto input_var = step_scopes[0]->parent().FindVar(inlinks[i]); + PADDLE_ENFORCE_NOT_NULL(input_var, "input link [%s] is not in scope.", + inlinks[i]); LoDTensor* input = input_var->GetMutable(); f::DDim dims = input->dims(); - PADDLE_ENFORCE(static_cast(dims[0]) == seq_len, - "all the inlinks must have same length"); + PADDLE_ENFORCE_EQ(static_cast(dims[0]), seq_len, + "all the inlinks be the same length"); f::DDim step_dims = slice_ddim(dims, 1, dims.size()); for (size_t j = 0; j < seq_len; j++) { Tensor* step_input = - step_scopes[j]->NewVar(inlinks[i].internal)->GetMutable(); + step_scopes[j]->NewVar(inlinks[i])->GetMutable(); if (!infer_shape_mode) { // The input of operators of each step is Tensor here. // Maybe need to modify Slice function. @@ -51,18 +52,17 @@ void SegmentInputs(const std::vector& step_scopes, } void ConcatOutputs(const std::vector& step_scopes, - const std::vector& outlinks, const size_t seq_len, - bool infer_shape_mode) { + const std::vector& outlinks, + const size_t seq_len, bool infer_shape_mode) { for (size_t i = 0; i < outlinks.size(); i++) { - auto output_var = step_scopes[0]->FindVar(outlinks[i].external); - PADDLE_ENFORCE(output_var != nullptr, "output link [%s] is not in scope.", - outlinks[i].external); + auto output_var = step_scopes[0]->parent().FindVar(outlinks[i]); + PADDLE_ENFORCE_NOT_NULL(output_var, "output link [%s] is not in scope.", + outlinks[i]); LoDTensor* output = output_var->GetMutable(); if (infer_shape_mode) { - auto step_scope_var = step_scopes[0]->FindVar(outlinks[i].internal); - PADDLE_ENFORCE(step_scope_var != nullptr, "%s not in scope", - outlinks[i].internal); + auto step_scope_var = step_scopes[0]->FindVar(outlinks[i]); + PADDLE_ENFORCE_NOT_NULL(step_scope_var, "%s not in scope", outlinks[i]); f::DDim step_dims = step_scope_var->template GetMutable()->dims(); std::vector dims_vec = vectorize(step_dims); @@ -71,9 +71,8 @@ void ConcatOutputs(const std::vector& step_scopes, } else { output->mutable_data(platform::CPUPlace()); for (size_t j = 0; j < seq_len; j++) { - LoDTensor* step_output = step_scopes[j] - ->FindVar(outlinks[i].internal) - ->GetMutable(); + LoDTensor* step_output = + step_scopes[j]->FindVar(outlinks[i])->GetMutable(); // TODO(luotao02) data type and platform::DeviceContext() should set // correctly (output->Slice(j, j + 1)) @@ -113,29 +112,9 @@ void InitArgument(const ArgumentName& name, Argument* arg, const framework::OperatorBase& op) { arg->step_scopes = op.Output(name.step_scopes); - auto inlinks = op.Inputs(name.inlinks); - auto inlink_alias = op.Attr>(name.inlink_alias); - PADDLE_ENFORCE(inlinks.size() == inlink_alias.size(), - "the size of inlinks and inlink_alias don't match:%d,%d", - inlinks.size(), inlink_alias.size()); - for (size_t i = 0; i < inlinks.size(); ++i) { - rnn::Link link; - link.external = inlinks[i]; - link.internal = inlink_alias[i]; - (arg->inlinks).push_back(link); - } + arg->inlinks = op.Inputs(name.inlinks); - auto outlinks = op.Outputs(name.outlinks); - auto outlink_alias = op.Attr>(name.outlink_alias); - PADDLE_ENFORCE(outlinks.size() == outlink_alias.size(), - "the size of outlinks and outlink_alias don't match:%d,%d", - outlinks.size(), outlink_alias.size()); - for (size_t i = 0; i < outlinks.size(); ++i) { - rnn::Link link; - link.external = outlinks[i]; - link.internal = outlink_alias[i]; - (arg->outlinks).push_back(link); - } + arg->outlinks = op.Outputs(name.outlinks); auto boot_memories = op.Inputs(name.boot_memories); diff --git a/paddle/operators/rnn/recurrent_op_utils.h b/paddle/operators/rnn/recurrent_op_utils.h index 17941c503cfcc83415b8bc635623a2c2ce2981c3..7dafe5d0088c4c8bf2cad163654e7e4f28eebe2e 100644 --- a/paddle/operators/rnn/recurrent_op_utils.h +++ b/paddle/operators/rnn/recurrent_op_utils.h @@ -41,18 +41,11 @@ struct MemoryAttr { std::string boot_var; }; -struct Link { - // input or output links name. - std::string internal; - // alias to avoid duplicate keys in scopes. - std::string external; -}; - struct Argument { std::string step_net; std::string step_scopes; - std::vector inlinks; - std::vector outlinks; + std::vector inlinks; + std::vector outlinks; std::vector memories; }; @@ -61,8 +54,6 @@ struct ArgumentName { std::string step_scopes; std::string inlinks; std::string outlinks; - std::string inlink_alias; // the alias of inlinks in step net. - std::string outlink_alias; // the alias of outlinks in step net. std::string memories; // the memory name std::string pre_memories; // the previous memory name std::string boot_memories; // the boot memory name @@ -72,15 +63,15 @@ struct ArgumentName { * Prepare inputs for each step net. */ void SegmentInputs(const std::vector& step_scopes, - const std::vector& inlinks, const size_t seq_len, - bool infer_shape_mode); + const std::vector& inlinks, + const size_t seq_len, bool infer_shape_mode); /** * Process outputs of step nets and merge to variables. */ void ConcatOutputs(const std::vector& step_scopes, - const std::vector& outlinks, const size_t seq_len, - bool infer_shape_mode); + const std::vector& outlinks, + const size_t seq_len, bool infer_shape_mode); void LinkMemories(const std::vector& step_scopes, const std::vector& memories, const size_t step_id, diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc deleted file mode 100644 index 992b19965e0ca9ce7dba1b8b3c5b7780af06eb45..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.cc +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/sigmoid_op.h" - -namespace paddle { -namespace operators { - -class SigmoidOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input(X) of SigmoidOp should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Y"), - "Output(Y) of SigmoidOp should not be null."); - - ctx.Output("Y")->Resize( - ctx.Input("X")->dims()); - } -}; - -class SigmoidOpMaker : public framework::OpProtoAndCheckerMaker { - public: - SigmoidOpMaker(framework::OpProto *proto, - framework::OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "sigmoid input"); - AddOutput("Y", "sigmoid output"); - AddComment("Sigmoid function"); - } -}; - -class SigmoidOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(const framework::InferShapeContext &ctx) const override { - ctx.Output(framework::GradVarName("X")) - ->Resize(ctx.Input("Y")->dims()); - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad, - ops::SigmoidOpGrad); -REGISTER_OP_CPU_KERNEL(sigmoid, - ops::SigmoidKernel); -REGISTER_OP_CPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); diff --git a/paddle/operators/sigmoid_op.h b/paddle/operators/sigmoid_op.h deleted file mode 100644 index b01a9b3f23283471f8846325075719ba0e75ed35..0000000000000000000000000000000000000000 --- a/paddle/operators/sigmoid_op.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; - -template -class SigmoidKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto input = context.Input("X"); - auto output = context.Output("Y"); - output->mutable_data(context.GetPlace()); - - // The clipping is used in Paddle's raw implenmention - auto X = EigenVector::Flatten(*input); - auto Y = EigenVector::Flatten(*output); - auto place = context.GetEigenDevice(); - - Y.device(place) = 1. / (1. + (-X).exp()); - } -}; - -template -class SigmoidGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto Y_t = context.Input("Y"); - auto dY_t = context.Input(framework::GradVarName("Y")); - auto dX_t = context.Output(framework::GradVarName("X")); - - dX_t->mutable_data(context.GetPlace()); - - auto dX = EigenVector::Flatten(*dX_t); - auto Y = EigenVector::Flatten(*Y_t); - auto dY = EigenVector::Flatten(*dY_t); - dX.device(context.GetEigenDevice()) = dY * Y * (1. - Y); - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..9ee6fff8db6a285a0314431e4e13b284c78c8a70 --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -0,0 +1,135 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/smooth_l1_loss_op.h" + +namespace paddle { +namespace operators { + +class SmoothL1LossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Y must be initialized."); + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "The shape of X and Y must be the same."); + PADDLE_ENFORCE_GE(x->dims().size(), 2, + "The tensor rank of X must be at least 2."); + auto* inside_weight = ctx.Input("InsideWeight"); + if (inside_weight) { + auto* outside_weight = ctx.Input("OutsideWeight"); + PADDLE_ENFORCE_NOT_NULL(outside_weight, + "If weights are provided, must specify both " + "inside and outside weights."); + PADDLE_ENFORCE_EQ(inside_weight->dims(), x->dims(), + "The shape of InsideWeight must be same as X."); + PADDLE_ENFORCE_EQ(outside_weight->dims(), x->dims(), + "The shape of OutsideWeight must be same as X."); + } + + auto* diff = ctx.Output("Diff"); + auto* out = ctx.Output("Out"); + diff->Resize(x->dims()); + // loss is a two-rank tensor + out->Resize({x->dims()[0], 1}); + } +}; + +template +class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SmoothL1LossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", + "The input tensor of smooth l1 loss op." + "The rank should be greater or equal to 2 with shape " + "[batch_size, value_dim1, value_dim2, ..., value_dimN]"); + AddInput("Y", + "The target tensor of smooth l1 loss op " + "with the same shape as X."); + AddInput("InsideWeight", + "Optional input tensor of smooth l1 loss op with the same shape " + "as X. If provided, the result of (X - Y) will be multiplied " + "by this tensor element by element."); + AddInput("OutsideWeight", + "Optinal input of smooth l1 loss op with the same shape as X." + "If provided, the output smooth l1 loss will be multiplied by " + "this tensor element by element."); + AddOutput("Diff", "Intermediate variable to cache InsideWeight*(X-Y).") + .AsIntermediate(); + AddOutput("Out", "Smooth l1 loss."); + AddAttr("sigma", + "Hyper parameter of smooth l1 loss op." + "A float scalar with default value 3.0.") + .SetDefault(3.0); + AddComment(R"DOC( +Compute smooth l1 loss for input and target. The operator take the 1st +dimension of input as batch size. For each instance, it will compute +smooth l1 loss element by element first and sum all losses to one value. +So the output shape is [batch_size, 1]. + +The equation is: +loss = 0.5 * (sigma * (x-y))^2 if abs(x - y) < 1 / sigma^2 + abs(x - y) - 0.5 / sigma^2 otherwise + +)DOC"); + } +}; + +class SmoothL1LossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + auto in_dims = ctx.Input("X")->dims(); + auto out_dims = + ctx.Input(framework::GradVarName("Out"))->dims(); + auto* x_grad = + ctx.Output(framework::GradVarName("X")); + auto* y_grad = + ctx.Output(framework::GradVarName("Y")); + + PADDLE_ENFORCE_GE(out_dims.size(), 2, + "The tensor rank of Input(Out@Grad) should be 2."); + PADDLE_ENFORCE_EQ(out_dims[0], in_dims[0], + "The 1st dimension of Input(Out@Grad) must be " + "same as input."); + PADDLE_ENFORCE_EQ(out_dims[1], 1, + "The 2nd dimension of Input(Out@Grad) must be 1."); + + if (x_grad) x_grad->Resize(in_dims); + if (y_grad) y_grad->Resize(in_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(smooth_l1_loss, ops::SmoothL1LossOp, + ops::SmoothL1LossOpMaker, smooth_l1_loss_grad, + ops::SmoothL1LossGradOp); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/sigmoid_op.cu b/paddle/operators/smooth_l1_loss_op.cu similarity index 73% rename from paddle/operators/sigmoid_op.cu rename to paddle/operators/smooth_l1_loss_op.cu index 1a50dfe14a7b9e2614aadb7729de9f9e461e9905..1c3172f43867741cd1f26979a366b2425f326321 100644 --- a/paddle/operators/sigmoid_op.cu +++ b/paddle/operators/smooth_l1_loss_op.cu @@ -13,11 +13,12 @@ limitations under the License. */ #define EIGEN_USE_GPU -#include "paddle/operators/sigmoid_op.h" -namespace ops = paddle::operators; +#include "paddle/operators/smooth_l1_loss_op.h" -REGISTER_OP_GPU_KERNEL(sigmoid, - ops::SigmoidKernel); +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); REGISTER_OP_GPU_KERNEL( - sigmoid_grad, ops::SigmoidGradKernel); + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h new file mode 100644 index 0000000000000000000000000000000000000000..0604fb5e1c2f17c702208520a1d23bd5c3c65b5d --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.h @@ -0,0 +1,182 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; +template +using EigenMatrix = framework::EigenMatrix; + +template +struct SmoothL1LossForward { + HOSTDEVICE SmoothL1LossForward(const T& sigma2) : sigma2(sigma2) {} + + HOSTDEVICE T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return 0.5 * val * val * sigma2; + } else { + return abs_val - 0.5 / sigma2; + } + } + + T sigma2; +}; + +template +class SmoothL1LossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* in2 = context.Input("InsideWeight"); + auto* in3 = context.Input("OutsideWeight"); + auto* out0 = context.Output("Diff"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto sigma = static_cast(context.Attr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in2 != nullptr) && (in3 != nullptr); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + auto diff = EigenVector::Flatten(*out0); + + diff.device(place) = x - y; + // multiply inside weight + if (has_weight) { + auto inside_weight = EigenVector::Flatten(*in2); + // cache diff, reused in bp + diff.device(place) = diff * inside_weight; + } + + auto in_counts = in0->numel(); + Tensor ptensor_errors; + ptensor_errors.mutable_data({static_cast(in_counts)}, + context.GetPlace()); + auto errors = EigenVector::Flatten(ptensor_errors); + // apply smooth l1 forward + errors.device(place) = diff.unaryExpr(SmoothL1LossForward(sigma2)); + + // multiply outside weight + if (has_weight) { + auto outside_weight = EigenVector::Flatten(*in3); + errors.device(place) = errors * outside_weight; + } + auto loss = EigenVector::Flatten(*out1); + // first dimension of 'X' is the number of samples + auto mat_dims = + framework::make_ddim({static_cast(in0->dims()[0]), + static_cast(in_counts / in0->dims()[0])}); + auto errors_mat_view = EigenMatrix::From(ptensor_errors, mat_dims); + loss.device(place) = errors_mat_view.sum(Eigen::array({{1}})); + } +}; + +template +struct SmoothL1LossBackward { + HOSTDEVICE SmoothL1LossBackward(const T& sigma2) : sigma2(sigma2) {} + + HOSTDEVICE T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return sigma2 * val; + } else { + return (0 < val) - (val < 0); + } + } + + T sigma2; +}; + +template +class SmoothL1LossGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("InsideWeight"); + auto* in1 = context.Input("OutsideWeight"); + auto* in2 = context.Input("Diff"); + auto* og = context.Input(framework::GradVarName("Out")); + auto sigma = static_cast(context.Attr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in0 != nullptr) && (in1 != nullptr); + + auto place = context.GetEigenDevice(); + + auto in_dims = in2->dims(); + auto counts = in2->numel(); + auto cols = counts / in_dims[0]; + auto mat_dims = framework::make_ddim( + {static_cast(in_dims[0]), static_cast(cols)}); + + Tensor ptensor_diff; + ptensor_diff.mutable_data({static_cast(counts)}, + context.GetPlace()); + auto diff = EigenVector::Flatten(ptensor_diff); + // apply smooth l1 backwoard + diff.device(place) = EigenVector::Flatten(*in2).unaryExpr( + SmoothL1LossBackward(sigma2)); + + // compute weights + Tensor ptensor_weights; + ptensor_weights.mutable_data(mat_dims, context.GetPlace()); + auto weights = EigenMatrix::From(ptensor_weights); + // initialize to 1.0 + weights.device(place) = weights.constant(static_cast(1.0)); + if (has_weight) { + auto inside_weight = EigenMatrix::From(*in0, mat_dims); + auto outside_weight = EigenMatrix::From(*in1, mat_dims); + weights.device(place) = inside_weight * outside_weight; + } + + // compute gradients + auto out_grad = EigenMatrix::From(*og); + auto diff_mat_view = EigenMatrix::From(ptensor_diff, mat_dims); + auto gradients = out_grad.broadcast( + Eigen::array({{1, static_cast(cols)}})) * + weights * diff_mat_view; + + auto* out0 = context.Output(framework::GradVarName("X")); + auto* out1 = context.Output(framework::GradVarName("Y")); + + if (out0) { + out0->mutable_data(context.GetPlace()); + auto x_grad = EigenMatrix::From(*out0, mat_dims); + x_grad.device(place) = gradients; + } + + if (out1) { + out1->mutable_data(context.GetPlace()); + auto y_grad = EigenMatrix::From(*out1, mat_dims); + y_grad.device(place) = -1 * gradients; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/strided_memcpy.h b/paddle/operators/strided_memcpy.h new file mode 100644 index 0000000000000000000000000000000000000000..c9dd80518424017d9834a2bf7aee14caa56c9d79 --- /dev/null +++ b/paddle/operators/strided_memcpy.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/operators/detail/strided_memcpy.h" + +namespace paddle { +namespace operators { + +// Strided memory copy from src to dst. +// +// The src and dst should be both on dev_ctx.GetPlace(), otherwise, there will +// be a segment fault. +// +// The stride of an array (also referred to as increment, pitch or step size) is +// the number of locations in memory between beginnings of successive array +// elements +// +// For example, for tensor like [1, 3, 300, 300]. If there is no padding, the +// stride is [270000, 90000, 300, 1]. +// +// NOTE: When use GPU, the memcpy is async. To sync memcpy, please invoke +// `dev_ctx.Wait()`. +template +inline void StridedMemcpy(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_dim, + const framework::DDim& dst_stride, T* dst) { + using namespace detail; + StridedCopyDimVisitor func(dev_ctx, src, src_stride, dst_stride, dst); + boost::apply_visitor(func, dst_dim); +} +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..05882a88738cfc9cc23480efe0afe504008377ca --- /dev/null +++ b/paddle/operators/strided_memcpy_test.cc @@ -0,0 +1,160 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/strided_memcpy.h" +#include "gtest/gtest.h" +#include "paddle/memory/memory.h" + +namespace paddle { +namespace operators { + +TEST(StridedMemcpy, CPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + framework::DDim src_stride({5, 1}); + + int dst[4]; + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CPUDeviceContext ctx; + StridedMemcpy(ctx, src + 1, src_stride, dst_dim, dst_stride, dst); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); +} + +TEST(StridedMemcpy, CPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + int dst[8]; + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CPUDeviceContext ctx; + + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst); + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst + 2); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } +} + +#ifndef PADDLE_ONLY_CPU +TEST(StridedMemcpy, GPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + framework::DDim src_stride({5, 1}); + + int dst[4]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CUDADeviceContext ctx(gpu0); + StridedMemcpy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, + gpu_dst); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +TEST(StridedMemcpy, GPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + int dst[8]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CUDADeviceContext ctx(gpu0); + + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst); + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, + gpu_dst + 2); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +#endif +} // namespace operators +} // namespace paddle \ No newline at end of file diff --git a/paddle/operators/transpose_op.cc b/paddle/operators/transpose_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..babf2f561c31d5436fe1611c576e6e7fc04401db --- /dev/null +++ b/paddle/operators/transpose_op.cc @@ -0,0 +1,119 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/transpose_op.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; + +class TransposeOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Output(Out) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + std::vector axis = ctx.Attr>("axis"); + size_t x_rank = x_dims.size(); + size_t axis_size = axis.size(); + + PADDLE_ENFORCE_EQ(x_rank, axis_size, + "the input tensor's rank(%d) " + "should be equal to the axis's size(%d)", + x_rank, axis_size); + + std::vector count(axis_size, 0); + for (size_t i = 0; i < axis_size; i++) { + PADDLE_ENFORCE( + axis[i] < static_cast(axis_size) && ++count[axis[i]] == 1, + "Each element of Attribute axis should be a unique value " + "range from 0 to (dims - 1), " + "where the dims is the axis's size"); + } + + framework::DDim out_dims(x_dims); + for (size_t i = 0; i < axis_size; i++) { + out_dims[i] = x_dims[axis[i]]; + } + ctx.Output("Out")->Resize(out_dims); + } +}; + +class TransposeOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TransposeOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "X", + "(Tensor)The input tensor, tensors with rank at most 6 are supported"); + AddOutput("Out", "(Tensor)The output tensor"); + AddAttr>( + "axis", + "(vector)a list of values, and the size of the list should be " + "the same with the input tensor rank, the tensor will " + "permute the axes according the the values given"); + AddComment(R"DOC( +The Tensor will be permuted according to the axis values given. +The op is very much like the numpy.transpose function in python +For example: + >> input = numpy.arange(6).reshape((2,3)) + >> input + array([[0, 1, 2], + [3, 4, 5]]) + >> axis = [1, 0] + >> output = input.transpose(axis) + >> output + array([[0, 3], + [1, 4], + [2, 5]]) +So, given a input tensor of shape(N, C, H, W) and the axis is {0, 2, 3, 1}, +the output tensor shape will be (N, H, W, C) +)DOC"); + } +}; + +class TransposeOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "Input(X) should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto x_dims = ctx.Input("X")->dims(); + auto *x_grad = + ctx.Output(framework::GradVarName("X")); + + if (x_grad) x_grad->Resize(x_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(transpose, ops::TransposeOp, ops::TransposeOpMaker, transpose_grad, + ops::TransposeOpGrad); +REGISTER_OP_CPU_KERNEL(transpose, + ops::TransposeKernel); +REGISTER_OP_CPU_KERNEL( + transpose_grad, + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.cu b/paddle/operators/transpose_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..af3f581462c919bbd2dd1067e536cc638f9c267d --- /dev/null +++ b/paddle/operators/transpose_op.cu @@ -0,0 +1,22 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/transpose_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(transpose, + ops::TransposeKernel); +REGISTER_OP_GPU_KERNEL( + transpose_grad, + ops::TransposeGradKernel); diff --git a/paddle/operators/transpose_op.h b/paddle/operators/transpose_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ea299dce72ad340b0a65ee50582dc156b5ad7abb --- /dev/null +++ b/paddle/operators/transpose_op.h @@ -0,0 +1,128 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +void EigenTranspose(const framework::ExecutionContext& context, + const framework::Tensor& in, framework::Tensor& out, + std::vector axis) { + Eigen::array permute; + for (int i = 0; i < Rank; i++) { + permute[i] = axis[i]; + } + auto in_dim = in.dims(); + auto out_dim = out.dims(); + + auto eigen_in = framework::EigenTensor::From(in); + auto eigen_out = framework::EigenTensor::From(out); + auto& dev = context.GetEigenDevice(); + eigen_out.device(dev) = eigen_in.shuffle(permute); +} + +template +class TransposeKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); + auto* out = context.Output("Out"); + out->mutable_data(context.GetPlace()); + + std::vector axis = context.Attr>("axis"); + int ndims = axis.size(); + switch (ndims) { + case 1: + EigenTranspose(context, *x, *out, axis); + break; + case 2: + EigenTranspose(context, *x, *out, axis); + break; + case 3: + EigenTranspose(context, *x, *out, axis); + break; + case 4: + EigenTranspose(context, *x, *out, axis); + break; + case 5: + EigenTranspose(context, *x, *out, axis); + break; + case 6: + EigenTranspose(context, *x, *out, axis); + break; + default: + PADDLE_THROW("Tensors with rank at most 6 are supported"); + } + } +}; + +template +class TransposeGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* out_grad = + context.Input(framework::GradVarName("Out")); + auto* x_grad = + context.Output(framework::GradVarName("X")); + if (x_grad) { + x_grad->mutable_data(context.GetPlace()); + + std::vector axis = context.Attr>("axis"); + std::vector reversed_axis(axis); + + for (size_t i = 0; i < axis.size(); i++) { + reversed_axis[axis[i]] = i; + } + + int ndims = axis.size(); + + switch (ndims) { + case 1: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 2: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 3: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 4: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 5: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + case 6: + EigenTranspose(context, *out_grad, *x_grad, + reversed_axis); + break; + default: + PADDLE_THROW("Tensors with rank at most 6 are supported"); + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt index 8b605e51c3f4ea38fc358ce054bb36fcc82063c4..daf519b91d623d4369774dc4e37dcb7b1733666b 100644 --- a/paddle/platform/CMakeLists.txt +++ b/paddle/platform/CMakeLists.txt @@ -24,4 +24,4 @@ cc_library(device_context SRCS device_context.cc DEPS memory buddy_allocator nv_test(device_context_test SRCS device_context_test.cc DEPS device_context gpu_info) nv_test(cudnn_helper_test SRCS cudnn_helper_test.cc DEPS dynload_cuda) -nv_test(transform_test SRCS transform_test.cu DEPS paddle_memory place) +nv_test(transform_test SRCS transform_test.cu DEPS paddle_memory place device_context) diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index ad212c5b2c47312743362db4926c80bf056e100d..93b472b41c8a4c3a2bfada9d4fbf0e9e1b0cc736 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -101,19 +101,17 @@ CUDADeviceContext::CUDADeviceContext(GPUPlace place) : place_(place) { eigen_stream_.reset(new EigenCudaStreamDevice()); eigen_stream_->Reinitialize(&stream_, place); eigen_device_.reset(new Eigen::GpuDevice(eigen_stream_.get())); + PADDLE_ENFORCE(dynload::cublasCreate(&cublas_handle_)); + PADDLE_ENFORCE(dynload::cublasSetStream(cublas_handle_, stream_)); + PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); + PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream_)); } CUDADeviceContext::~CUDADeviceContext() { SetDeviceId(place_.device); Wait(); - if (cublas_handle_) { - PADDLE_ENFORCE(dynload::cublasDestroy(cublas_handle_)); - } - - if (cudnn_handle_) { - PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); - } - + PADDLE_ENFORCE(dynload::cublasDestroy(cublas_handle_)); + PADDLE_ENFORCE(dynload::cudnnDestroy(cudnn_handle_)); eigen_stream_.reset(); eigen_device_.reset(); PADDLE_ENFORCE(cudaStreamDestroy(stream_)); @@ -129,25 +127,13 @@ Eigen::GpuDevice* CUDADeviceContext::eigen_device() const { return eigen_device_.get(); } -cublasHandle_t CUDADeviceContext::cublas_handle() { - if (!cublas_handle_) { - SetDeviceId(place_.device); - PADDLE_ENFORCE(dynload::cublasCreate(&cublas_handle_)); - PADDLE_ENFORCE(dynload::cublasSetStream(cublas_handle_, stream_)); - } +cublasHandle_t CUDADeviceContext::cublas_handle() const { return cublas_handle_; } -cudnnHandle_t CUDADeviceContext::cudnn_handle() { - if (!cudnn_handle_) { - SetDeviceId(place_.device); - PADDLE_ENFORCE(dynload::cudnnCreate(&cudnn_handle_)); - PADDLE_ENFORCE(dynload::cudnnSetStream(cudnn_handle_, stream_)); - } - return cudnn_handle_; -} +cudnnHandle_t CUDADeviceContext::cudnn_handle() const { return cudnn_handle_; } -cudaStream_t CUDADeviceContext::stream() { return stream_; } +cudaStream_t CUDADeviceContext::stream() const { return stream_; } #endif // PADDLE_ONLY_CPU diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index 11528e1194e4516891034fa8febdac3ba6eed204..a106592e454e21c46cd2f87f1bbf6694955d6e23 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -67,16 +67,14 @@ class CUDADeviceContext : public DeviceContext { /*! \brief Return eigen device in the device context. */ Eigen::GpuDevice* eigen_device() const; - // clang-format off /*! \brief Return cublas handle in the device context. */ - cublasHandle_t cublas_handle(); + cublasHandle_t cublas_handle() const; /*! \brief Return cudnn handle in the device context. */ - cudnnHandle_t cudnn_handle(); + cudnnHandle_t cudnn_handle() const; /*! \brief Return cuda stream in the device context. */ - cudaStream_t stream(); - // clang-format on + cudaStream_t stream() const; private: GPUPlace place_; @@ -84,11 +82,9 @@ class CUDADeviceContext : public DeviceContext { std::unique_ptr eigen_device_; std::unique_ptr eigen_stream_; - // clang-format off - cudaStream_t stream_{nullptr}; - cudnnHandle_t cudnn_handle_{nullptr}; - cublasHandle_t cublas_handle_{nullptr}; - // clang-format on + cudaStream_t stream_; + cudnnHandle_t cudnn_handle_; + cublasHandle_t cublas_handle_; }; #endif diff --git a/paddle/platform/transform.h b/paddle/platform/transform.h index 3ee4acd29660f201d318ce6d39baa6f3999ae274..f196868c725cbb91b3df710260c5b60f14d53f37 100644 --- a/paddle/platform/transform.h +++ b/paddle/platform/transform.h @@ -14,6 +14,7 @@ #pragma once +#include "paddle/platform/device_context.h" #include "paddle/platform/enforce.h" #include "paddle/platform/hostdevice.h" #include "paddle/platform/place.h" @@ -21,46 +22,78 @@ #include #include #ifdef __NVCC__ +#include #include #include "paddle/platform/details/device_ptr_cast.h" #endif namespace paddle { namespace platform { + // Transform on host or device. It provides the same API in std library. -template -void Transform(Place place, InputIter first, InputIter last, OutputIter result, - UnaryOperation op) { - if (is_cpu_place(place)) { +template +struct Transform { + template + void operator()(const DeviceContext& context, InputIter first, InputIter last, + OutputIter result, UnaryOperation op); + + template + void operator()(const DeviceContext& context, InputIter1 first1, + InputIter1 last1, InputIter2 first2, OutputIter result, + BinaryOperation op); +}; + +template <> +struct Transform { + template + void operator()(const DeviceContext& context, InputIter first, InputIter last, + OutputIter result, UnaryOperation op) { + auto place = context.GetPlace(); + PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first, last, result, op); - } else { -#ifdef __NVCC__ - using namespace details; - thrust::transform(DevPtrCast(first), DevPtrCast(last), DevPtrCast(result), - op); -#else - PADDLE_THROW("Do not invoke `Transform` in .cc file"); -#endif } -} -template -void Transform(Place place, InputIter1 first1, InputIter1 last1, - InputIter2 first2, OutputIter result, BinaryOperation op) { - if (is_cpu_place(place)) { + template + void operator()(const DeviceContext& context, InputIter1 first1, + InputIter1 last1, InputIter2 first2, OutputIter result, + BinaryOperation op) { + auto place = context.GetPlace(); + PADDLE_ENFORCE(is_cpu_place(place), "It must use CPU place."); std::transform(first1, last1, first2, result, op); - } else { + } +}; + #ifdef __NVCC__ - using namespace details; - thrust::transform(DevPtrCast(first1), DevPtrCast(last1), DevPtrCast(first2), - DevPtrCast(result), op); -#else - PADDLE_THROW("Do not invoke `Transform` in .cc file"); -#endif +template <> +struct Transform { + template + void operator()(const DeviceContext& context, InputIter first, InputIter last, + OutputIter result, UnaryOperation op) { + auto place = context.GetPlace(); + PADDLE_ENFORCE(is_gpu_place(place), "It must use GPU place."); + auto& ctx = reinterpret_cast(context); + thrust::transform(thrust::cuda::par.on(ctx.stream()), + details::DevPtrCast(first), details::DevPtrCast(last), + details::DevPtrCast(result), op); + } + + template + void operator()(const DeviceContext& context, InputIter1 first1, + InputIter1 last1, InputIter2 first2, OutputIter result, + BinaryOperation op) { + auto place = context.GetPlace(); + PADDLE_ENFORCE(is_gpu_place(place), "It must use GPU place."); + auto& ctx = reinterpret_cast(context); + thrust::transform(thrust::cuda::par.on(ctx.stream()), + details::DevPtrCast(first1), details::DevPtrCast(last1), + details::DevPtrCast(first2), details::DevPtrCast(result), + op); } }; +#endif } // namespace platform } // namespace paddle diff --git a/paddle/platform/transform_test.cu b/paddle/platform/transform_test.cu index 600fed8f45077a6fee91f295aa854153c9cf9c01..c76cab80e4b0e8df98a7be15f86699cfb6f93af2 100644 --- a/paddle/platform/transform_test.cu +++ b/paddle/platform/transform_test.cu @@ -15,6 +15,7 @@ #include #include "paddle/memory/memcpy.h" #include "paddle/memory/memory.h" +#include "paddle/platform/hostdevice.h" #include "paddle/platform/transform.h" template @@ -36,8 +37,10 @@ class Multiply { TEST(Transform, CPUUnary) { using namespace paddle::platform; + CPUDeviceContext ctx; float buf[4] = {0.1, 0.2, 0.3, 0.4}; - Transform(CPUPlace(), buf, buf + 4, buf, Scale(10)); + Transform trans; + trans(ctx, buf, buf + 4, buf, Scale(10)); for (int i = 0; i < 4; ++i) { ASSERT_NEAR(buf[i], static_cast(i + 1), 1e-5); } @@ -47,10 +50,13 @@ TEST(Transform, GPUUnary) { using namespace paddle::platform; using namespace paddle::memory; GPUPlace gpu0(0); + CUDADeviceContext ctx(gpu0); float cpu_buf[4] = {0.1, 0.2, 0.3, 0.4}; float* gpu_buf = static_cast(Alloc(gpu0, sizeof(float) * 4)); Copy(gpu0, gpu_buf, CPUPlace(), cpu_buf, sizeof(cpu_buf)); - Transform(gpu0, gpu_buf, gpu_buf + 4, gpu_buf, Scale(10)); + Transform trans; + trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, Scale(10)); + ctx.Wait(); Copy(CPUPlace(), cpu_buf, gpu0, gpu_buf, sizeof(cpu_buf)); Free(gpu0, gpu_buf); for (int i = 0; i < 4; ++i) { @@ -62,7 +68,9 @@ TEST(Transform, CPUBinary) { using namespace paddle::platform; using namespace paddle::memory; int buf[4] = {1, 2, 3, 4}; - Transform(CPUPlace(), buf, buf + 4, buf, buf, Multiply()); + Transform trans; + CPUDeviceContext ctx; + trans(ctx, buf, buf + 4, buf, buf, Multiply()); for (int i = 0; i < 4; ++i) { ASSERT_EQ((i + 1) * (i + 1), buf[i]); } @@ -73,12 +81,15 @@ TEST(Transform, GPUBinary) { using namespace paddle::memory; int buf[4] = {1, 2, 3, 4}; GPUPlace gpu0(0); + CUDADeviceContext ctx(gpu0); int* gpu_buf = static_cast(Alloc(gpu0, sizeof(buf))); Copy(gpu0, gpu_buf, CPUPlace(), buf, sizeof(buf)); - Transform(gpu0, gpu_buf, gpu_buf + 4, gpu_buf, gpu_buf, Multiply()); + Transform trans; + trans(ctx, gpu_buf, gpu_buf + 4, gpu_buf, gpu_buf, Multiply()); + ctx.Wait(); Copy(CPUPlace(), buf, gpu0, gpu_buf, sizeof(buf)); Free(gpu0, gpu_buf); for (int i = 0; i < 4; ++i) { ASSERT_EQ((i + 1) * (i + 1), buf[i]); } -} \ No newline at end of file +} diff --git a/paddle/pserver/CMakeLists.txt b/paddle/pserver/CMakeLists.txt index 2245c7d88ca74922f9919db91977dfa6cb3ca468..ccfc0e76020c7b4f54a493cc4048e7571379ec1a 100644 --- a/paddle/pserver/CMakeLists.txt +++ b/paddle/pserver/CMakeLists.txt @@ -45,14 +45,18 @@ add_dependencies(paddle_pserver paddle_proto ${external_project_dependencies}) set(PSERVER_MAIN_SOURCES ParameterServer2Main.cpp) -add_executable(paddle_pserver_main - ${PSERVER_MAIN_SOURCES}) -link_paddle_exe(paddle_pserver_main) if(WITH_TESTING) add_subdirectory(test) endif() -install(TARGETS paddle_pserver_main - RUNTIME DESTINATION opt/paddle/bin - PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ - GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ) -set_target_properties(paddle_pserver_main PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) + +if(NOT WITH_C_API) + add_executable(paddle_pserver_main ${PSERVER_MAIN_SOURCES}) + link_paddle_exe(paddle_pserver_main) + + install(TARGETS paddle_pserver_main + RUNTIME DESTINATION opt/paddle/bin + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ) + + set_target_properties(paddle_pserver_main PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() diff --git a/paddle/scripts/travis/build_ios.sh b/paddle/scripts/travis/build_ios.sh new file mode 100755 index 0000000000000000000000000000000000000000..dee7cf7cbbcccffd727002108ae7f6b6ee2fbba8 --- /dev/null +++ b/paddle/scripts/travis/build_ios.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +# Create the build directory for CMake. +mkdir -p $TRAVIS_BUILD_DIR/build_ios +cd $TRAVIS_BUILD_DIR/build_ios + +# Compile paddle binaries +cmake -DCMAKE_SYSTEM_NAME=iOS \ + -DIOS_PLATFORM=OS \ + -DCMAKE_OSX_ARCHITECTURES="arm64" \ + -DWITH_C_API=ON \ + -DUSE_EIGEN_FOR_BLAS=ON \ + -DWITH_TESTING=OFF \ + -DWITH_SWIG_PY=OFF \ + -DWITH_STYLE_CHECK=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + .. + +make -j 2 diff --git a/paddle/scripts/travis/check_style.sh b/paddle/scripts/travis/check_style.sh index ec499a839ac6593bac788f4cca5e33afbed73010..cb483b0ffc0a1d99978508bc16464a7716d2bac2 100755 --- a/paddle/scripts/travis/check_style.sh +++ b/paddle/scripts/travis/check_style.sh @@ -8,6 +8,12 @@ function abort(){ trap 'abort' 0 set -e +# install glide +curl https://glide.sh/get | bash +eval "$(GIMME_GO_VERSION=1.8.3 gimme)" +go get -u github.com/alecthomas/gometalinter +gometalinter --install + cd $TRAVIS_BUILD_DIR export PATH=/usr/bin:$PATH pre-commit install diff --git a/paddle/trainer/CMakeLists.txt b/paddle/trainer/CMakeLists.txt index eac0584d30958ab78a935d89d217a4876fb07a19..3d471a0c01ca17cb98272159baf6d489c18824d5 100644 --- a/paddle/trainer/CMakeLists.txt +++ b/paddle/trainer/CMakeLists.txt @@ -50,22 +50,22 @@ macro(add_paddle_exe TARGET_NAME) link_paddle_exe(${TARGET_NAME}) endmacro() -add_paddle_exe(paddle_trainer - TrainerMain.cpp) - -add_paddle_exe(paddle_merge_model - MergeModel.cpp) - if(WITH_TESTING) - add_subdirectory(tests) + add_subdirectory(tests) endif() -install(TARGETS paddle_trainer paddle_merge_model - RUNTIME DESTINATION opt/paddle/bin - PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ - GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ) -set_target_properties(paddle_trainer PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) -set_target_properties(paddle_merge_model PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) +if(NOT WITH_C_API) + add_paddle_exe(paddle_trainer TrainerMain.cpp) + add_paddle_exe(paddle_merge_model MergeModel.cpp) + + install(TARGETS paddle_trainer paddle_merge_model + RUNTIME DESTINATION opt/paddle/bin + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ + GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ) + + set_target_properties(paddle_trainer PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) + set_target_properties(paddle_merge_model PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) +endif() if(APPLE) set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security") @@ -73,6 +73,8 @@ endif() if(WITH_GOLANG) add_dependencies(paddle_trainer_lib paddle_pserver_cclient) - target_link_libraries(paddle_trainer paddle_pserver_cclient) target_link_libraries(paddle_trainer_lib paddle_pserver_cclient) + if(NOT WITH_C_API) + target_link_libraries(paddle_trainer paddle_pserver_cclient) + endif() endif(WITH_GOLANG) diff --git a/paddle/utils/Excepts.h b/paddle/utils/Excepts.h index 5c2c504f53a586f2991ccfae891991465fdb39b6..0add66da7464293795927431daf0e90359f40b52 100644 --- a/paddle/utils/Excepts.h +++ b/paddle/utils/Excepts.h @@ -17,7 +17,8 @@ limitations under the License. */ #include -#if defined(__APPLE__) || defined(__OSX__) +#if (defined(__APPLE__) || defined(__OSX__)) && !defined(__arm__) && \ + !defined(__aarch64__) int fegetexcept(void); int feenableexcept(unsigned int excepts); diff --git a/paddle/utils/arch/linux/Locks.cpp b/paddle/utils/arch/linux/Locks.cpp index 3a0903d1f268cf0132da3de43396391219edf004..a4e6c8f7b8397adc262588612c250bac5ef5eaa6 100644 --- a/paddle/utils/arch/linux/Locks.cpp +++ b/paddle/utils/arch/linux/Locks.cpp @@ -40,6 +40,8 @@ void Semaphore::wait() { sem_wait(&m->sem); } void Semaphore::post() { sem_post(&m->sem); } +/// SpinLockPrivate + #ifdef PADDLE_USE_PTHREAD_SPINLOCK class SpinLockPrivate { @@ -79,6 +81,8 @@ SpinLock::~SpinLock() { delete m; } void SpinLock::lock() { m->lock(); } void SpinLock::unlock() { m->unlock(); } +/// ThreadBarrierPrivate + #ifdef PADDLE_USE_PTHREAD_BARRIER class ThreadBarrierPrivate { @@ -136,6 +140,8 @@ public: #endif +/// ThreadBarrier + ThreadBarrier::ThreadBarrier(int count) : m(new ThreadBarrierPrivate(count)) {} ThreadBarrier::~ThreadBarrier() { delete m; } void ThreadBarrier::wait() { m->wait(); } diff --git a/paddle/utils/arch/osx/Excepts.cpp b/paddle/utils/arch/osx/Excepts.cpp index c8e904d8f9fe29e51447994af43dc62bf3514306..42ecaa06d256c9d259a20c648626605d77ce0308 100644 --- a/paddle/utils/arch/osx/Excepts.cpp +++ b/paddle/utils/arch/osx/Excepts.cpp @@ -14,7 +14,8 @@ limitations under the License. */ #include "paddle/utils/Excepts.h" -#if defined(__APPLE__) || defined(__OSX__) +#if (defined(__APPLE__) || defined(__OSX__)) && !defined(__arm__) && \ + !defined(__aarch64__) int fegetexcept(void) { static fenv_t fenv; diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 7c32eb0069f4075d72cd4c3654c83e3d5c98fb1c..0f57b81966647ca5c6f5cd2e5518d2d34942a549 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1565,6 +1565,10 @@ class LayerBase(object): self.config = g_config.model_config.layers.add() assert isinstance(self.config, LayerConfig) + use_mkldnn = bool(int(g_command_config_args.get("use_mkldnn", 0))) + mkldnn_acts = ['relu', 'tanh'] + if use_mkldnn and active_type in mkldnn_acts: + active_type = "mkldnn_" + active_type self.config.name = name self.config.type = type self.config.active_type = active_type diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 8c7d1738ad9753eb7afb27e893f979f8bce70a0d..c97e6c0a36774caaa4fd8f8130220849975451a0 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -781,11 +781,11 @@ class MixedLayerType(LayerOutput): :type size: int :param act: activation type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will - get a default Bias. - :type bias_attr: ParameterAttribute or None means has bias. Any other - type means no bias. + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ @@ -881,10 +881,11 @@ def mixed_layer(size=0, then this function will just return layer's name. :param act: Activation Type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: The extra layer config. Default is None. :type layer_attr: ExtraLayerAttribute :return: MixedLayerType object can add inputs or layer name. @@ -920,7 +921,7 @@ def data_layer(name, size, depth=None, height=None, width=None, data = data_layer(name="input", size=1000) - :param name: Name of this data layer. + :param name: The name of this layer. It is optional. :type name: basestring :param size: Size of this data layer. :type size: int @@ -960,7 +961,7 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): """ Define a embedding Layer. - :param name: Name of this embedding layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer for this embedding. NOTE: must be Index Data. :type input: LayerOutput @@ -1015,7 +1016,7 @@ def fc_layer(input, with mixed_layer(size=1024) as fc: fc += full_matrix_projection(input=layer) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple @@ -1025,10 +1026,11 @@ def fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute|list. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -1065,7 +1067,7 @@ def printer_layer(input, format=None, name=None): """ Print the output value of input layers. This layer is useful for debugging. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple @@ -1103,7 +1105,7 @@ def priorbox_layer(input, """ Compute the priorbox and set the variance. This layer is necessary for ssd. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -1152,7 +1154,7 @@ def multibox_loss_layer(input_loc, """ Compute the location loss and the confidence loss for ssd. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input_loc: The input predict locations. :type input_loc: LayerOutput | List of LayerOutput @@ -1224,10 +1226,10 @@ def detection_output_layer(input_loc, name=None): """ Apply the NMS to the output of network and compute the predict bounding - box location. The output of this layer could be None if there is no valid - bounding box. + box location. The output's shape of this layer could be zero if there is + no valid bounding box. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input_loc: The input predict locations. :type input_loc: LayerOutput | List of LayerOutput. @@ -1299,7 +1301,7 @@ def cross_channel_norm_layer(input, name=None, param_attr=None): a conv layer's output and scale the output by a group of trainable factors which dimensions equal to the channel's number. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -1364,7 +1366,7 @@ def pooling_layer(input, :param agg_level: AggregateLevel.TO_NO_SEQUENCE or AggregateLevel.TO_SEQUENCE :type agg_level: AggregateLevel - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layer name. :type input: LayerOutput @@ -1373,8 +1375,11 @@ def pooling_layer(input, :type pooling_type: BasePoolingType|None :param stride: The step size between successive pooling regions. :type stride: Int - :param bias_attr: Bias parameter attribute. False if no bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: The Extra Attributes for layer, such as dropout. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -1471,10 +1476,11 @@ def lstmemory(input, :type gate_act: BaseActivation :param state_act: state activation type, TanhActivation by default. :type state_act: BaseActivation - - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute @@ -1596,9 +1602,11 @@ def grumemory(input, This activation affects the :math:`z_t` and :math:`r_t`. It is the :math:`\\sigma` in the above formula. :type gate_act: BaseActivation - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute @@ -1657,7 +1665,7 @@ def last_seq(input, seq = last_seq(input=layer) :param agg_level: Aggregated level - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -1713,7 +1721,7 @@ def first_seq(input, seq = first_seq(input=layer) :param agg_level: aggregation level - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -1792,11 +1800,13 @@ def expand_layer(input, :type input: LayerOutput :param expand_as: Expand as this layer's sequence info. :type expand_as: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param expand_level: whether input layer is timestep(default) or sequence. :type expand_level: ExpandLevel :param layer_attr: extra layer attributes. @@ -1849,7 +1859,7 @@ def repeat_layer(input, :type input: LayerOutput :param num_repeats: Repeat the input so many times :type num_repeats: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :param as_row_vector: True for treating input as row vector and repeating in the column direction. This is equivalent to apply concat_layer() with num_repeats same input. @@ -1908,16 +1918,17 @@ def seq_reshape_layer(input, :type input: LayerOutput :param reshape_size: the size of reshaped sequence. :type reshape_size: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param act: Activation type. :type act: BaseActivation :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ @@ -1960,7 +1971,7 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): :type input: list|tuple :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2065,7 +2076,7 @@ def power_layer(input, weight, name=None, layer_attr=None): :type input: LayerOutput :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2109,7 +2120,7 @@ def scaling_layer(input, weight, name=None, layer_attr=None): :type input: LayerOutput :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2147,7 +2158,7 @@ def trans_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2187,7 +2198,7 @@ def rotate_layer(input, height, width, name=None, layer_attr=None): :type input: LayerOutput :param height: The height of the sample matrix :type height: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2232,7 +2243,7 @@ def cos_sim(a, b, scale=1, size=1, name=None, layer_attr=None): cos = cos_sim(a=layer1, b=layer2, size=3) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: input layer a :type a: LayerOutput @@ -2299,11 +2310,13 @@ def hsigmoid(input, :type label: LayerOutput :param num_classes: number of classes. :type num_classes: int|None - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring - :param bias_attr: Bias attribute. None means default bias. - False means no bias. - :type bias_attr: ParameterAttribute|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. None means default parameter. :type param_attr: ParameterAttribute|None :param layer_attr: Extra Layer Attribute. @@ -2411,7 +2424,7 @@ def img_conv_layer(input, bias_attr=False, act=ReluActivation()) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Layer Input. :type input: LayerOutput @@ -2442,9 +2455,11 @@ def img_conv_layer(input, :type dilation: int|tuple|list :param dilation_y: The y dimension of the dilation. :type dilation_y: int - :param bias_attr: Convolution bias attribute. None means default bias. - False means no bias. - :type bias_attr: ParameterAttribute|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param num_channels: number of input channels. If None will be set automatically from previous output. :type num_channels: int @@ -2835,7 +2850,7 @@ def spp_layer(input, num_channels=16, pool_type=MaxPooling()) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: layer's input. :type input: LayerOutput @@ -2929,7 +2944,7 @@ def img_cmrnorm_layer(input, norm = img_cmrnorm_layer(input=net, size=5) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: None|basestring :param input: layer's input. :type input: LayerOutput @@ -2992,7 +3007,7 @@ def batch_norm_layer(input, norm = batch_norm_layer(input=net, act=ReluActivation()) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: batch normalization input. Better be linear activation. Because there is an activation inside batch_normalization. @@ -3016,7 +3031,7 @@ def batch_norm_layer(input, :type num_channels: int :param bias_attr: :math:`\\beta`, better be zero when initialize. So the initial_std=0, initial_mean=1 is best practice. - :type bias_attr: ParameterAttribute + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: :math:`\\gamma`, better be one when initialize. So the initial_std=0, initial_mean=1 is best practice. :type param_attr: ParameterAttribute @@ -3091,7 +3106,7 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -3127,7 +3142,7 @@ def row_l2_norm_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -3179,16 +3194,18 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): dropout here. Please refer to dropout_layer for details. - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layers. It could be a LayerOutput or list/tuple of LayerOutput. :type input: LayerOutput|list|tuple :param act: Activation Type, default is tanh. :type act: BaseActivation - :param bias_attr: Bias attribute. If False, means no bias. None is default - bias. - :type bias_attr: ParameterAttribute|bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -3237,7 +3254,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): concat = concat_layer(input=[layer1, layer2]) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layers or projections :type input: list|tuple|collections.Sequence @@ -3330,7 +3347,7 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, concat = seq_concat_layer(a=layer1, b=layer2) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param a: input sequence layer :type a: LayerOutput @@ -3340,10 +3357,11 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, :type act: BaseActivation :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ @@ -3506,7 +3524,7 @@ def lstm_step_layer(input, output is :math:`o_t`, whose name is 'state' and can use :code:`get_output_layer` to extract this output. - :param name: Layer's name. + :param name: The name of this layer. It is optional. :type name: basestring :param size: Layer's size. NOTE: lstm layer's size, should be equal to :code:`input.size/4`, and should be equal to @@ -3524,8 +3542,11 @@ def lstm_step_layer(input, :param state_act: State Activation Type. Default is sigmoid, and should be sigmoid only. :type state_act: BaseActivation - :param bias_attr: Bias Attribute. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -3576,9 +3597,13 @@ def gru_step_layer(input, :param output_mem: :param size: :param act: - :param name: + :param name: The name of this layer. It is optional. :param gate_act: - :param bias_attr: + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: the parameter_attribute for transforming the output_mem from previous step. :param layer_attr: @@ -3632,10 +3657,14 @@ def gru_step_naive_layer(input, :param input: :param output_mem: :param size: - :param name: + :param name: The name of this layer. It is optional. :param act: :param gate_act: - :param bias_attr: + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: :param layer_attr: :return: @@ -3691,7 +3720,7 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): output besides the default one, please use get_output_layer first to get the output from input. - :param name: Layer's name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: get output layer's input. And this layer should contains multiple outputs. @@ -3757,11 +3786,14 @@ def recurrent_layer(input, :type input: LayerOutput :param act: activation. :type act: BaseActivation - :param bias_attr: bias attribute. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: parameter attribute. :type param_attr: ParameterAttribute - :param name: name of the layer + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -4000,7 +4032,7 @@ def maxid_layer(input, name=None, layer_attr=None): :param input: Input layer name. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -4033,7 +4065,7 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): out_prod = out_prod_layer(input1=vec1, input2=vec2) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input1: The first input layer name. :type input: LayerOutput @@ -4074,7 +4106,7 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): eos = eos_layer(input=layer, eos_id=id) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -4265,7 +4297,7 @@ def square_error_cost(input, cost = \\sum_{i=1}^N(t_i-y_i)^2 - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Network prediction. :type input: LayerOutput @@ -4307,7 +4339,7 @@ def classification_cost(input, """ classification cost Layer. - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layer name. network output. :type input: LayerOutput @@ -4611,7 +4643,7 @@ def pad_layer(input, :type pad_w: list|None :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -4679,7 +4711,7 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): conv_shift = conv_shift_layer(a=layer1, b=layer2) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: Input layer a. :type a: LayerOutput @@ -4735,7 +4767,7 @@ def tensor_layer(a, tensor = tensor_layer(a=layer1, b=layer2, size=1000) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: Input layer a. :type a: LayerOutput @@ -4747,10 +4779,11 @@ def tensor_layer(a, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -4797,7 +4830,7 @@ def selective_fc_layer(input, sel_fc = selective_fc_layer(input=input, size=128, act=TanhActivation()) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput|list|tuple @@ -4811,10 +4844,11 @@ def selective_fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -4870,7 +4904,7 @@ def sampling_id_layer(input, name=None, layer_attr=None): :param input: The input layer. :type input: LayerOutput - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -4908,7 +4942,7 @@ def slope_intercept_layer(input, :param input: The input layer. :type input: LayerOutput - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param slope: the scale factor. :type slope: float. @@ -4972,7 +5006,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): :type vectors: LayerOutput :param size: the dimension of this layer. :type size: int - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -5055,7 +5089,7 @@ def block_expand_layer(input, :type padding_x: int :param padding_y: The padding size in vertical direction. :type padding_y: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -5124,7 +5158,7 @@ def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None): :type num_channels: int|None :param groups: The group number of input layer. :type groups: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute @@ -5188,7 +5222,7 @@ def ctc_layer(input, :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer + :param name: The name of this layer. It is optional. :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool @@ -5265,7 +5299,7 @@ def warp_ctc_layer(input, :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: basestring|None :param blank: the 'blank' label used in ctc :type blank: int @@ -5329,7 +5363,7 @@ def crf_layer(input, :type weight: LayerOutput :param param_attr: Parameter attribute. None means default attribute :type param_attr: ParameterAttribute - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -5399,7 +5433,7 @@ def crf_decoding_layer(input, :type label: LayerOutput or None :param param_attr: Parameter attribute. None means default attribute :type param_attr: ParameterAttribute - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -5458,9 +5492,9 @@ def nce_layer(input, param_attr=[attr1, attr2], weight=layer3, num_classes=3, neg_distribution=[0.1,0.3,0.6]) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring - :param input: input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput. :type input: LayerOutput|list|tuple|collections.Sequence :param label: label layer :type label: LayerOutput @@ -5478,8 +5512,11 @@ def nce_layer(input, A uniform distribution will be used if not provided. If not None, its length must be equal to num_classes. :type neg_distribution: list|tuple|collections.Sequence|None - :param bias_attr: Bias parameter attribute. True if no bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: layer name. @@ -5594,7 +5631,7 @@ def rank_cost(left, :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -5648,7 +5685,7 @@ def lambda_cost(input, :param score: The 2nd input. Score of each sample. :type input: LayerOutput :param NDCG_num: The size of NDCG (Normalized Discounted Cumulative Gain), - e.g., 5 for NDCG@5. It must be less than for equal to the + e.g., 5 for NDCG@5. It must be less than or equal to the minimum size of lists. :type NDCG_num: int :param max_sort_size: The size of partial sorting in calculating gradient. @@ -5659,7 +5696,7 @@ def lambda_cost(input, than the size of a list, the algorithm will sort the entire list of get gradient. :type max_sort_size: int - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -5703,7 +5740,7 @@ def cross_entropy(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param coeff: The cost is multiplied with coeff. The coefficient affects the gradient in the backward. @@ -5751,7 +5788,7 @@ def cross_entropy_with_selfnorm(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. :type coeff: float. @@ -5791,7 +5828,7 @@ def sum_cost(input, name=None, layer_attr=None): :param input: The first input layer. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -5836,7 +5873,7 @@ def huber_regression_cost(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param delta: The difference between the observed and predicted values. :type delta: float. @@ -5886,7 +5923,7 @@ def huber_classification_cost(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. :type coeff: float. @@ -5929,7 +5966,7 @@ def multi_binary_label_cross_entropy(input, :type input: LayerOutput :param label: The input label. :type input: LayerOutput - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -6034,9 +6071,9 @@ def cross_entropy_over_beam(input, name=None): ]) - :param input: input beams for this layer. + :param input: Input beams for this layer. :type input: BeamInput - :param name: input beams for this layer. + :param name: The name of this layer. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6097,7 +6134,7 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None): :type input: LayerOutput :param label: The input label. :type input: LayerOutput - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is optional. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -6145,7 +6182,7 @@ def multiplex_layer(input, name=None, layer_attr=None): :param input: Input layers. :type input: list of LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -6176,12 +6213,21 @@ def multiplex_layer(input, name=None, layer_attr=None): @wrap_name_default("dropout") def dropout_layer(input, dropout_rate, name=None): """ - @TODO(yuyang18): Add comments. - :param name: - :param input: - :param dropout_rate: - :return: + The example usage is: + + .. code-block:: python + + dropout = dropout_layer(input=input_layer, dropout_rate=0.5) + + :param name: The name of this layer. It is optional. + :type name: basestring + :param input: The input layer. + :type input: LayerOutput + :param dropout_rate: The probability of dropout. + :type dropout_rate: float + :return: LayerOutput object. + :rtype: LayerOutput """ return addto_layer( name=name, @@ -6204,7 +6250,7 @@ def row_conv_layer(input, """ The row convolution is called lookahead convolution. It is firstly - introduced in paper of `Deep Speech 2: End-toEnd Speech Recognition + introduced in paper of `Deep Speech 2: End-to-End Speech Recognition in English and Mandarin `_ . The bidirectional RNN that learns representation for a sequence by @@ -6212,9 +6258,9 @@ def row_conv_layer(input, However, unlike unidirectional RNNs, bidirectional RNNs are challenging to deploy in an online and low-latency setting. The lookahead convolution incorporates information from future subsequences in a computationally - efficient manner to improve unidirectional recurrent neural networks. + efficient manner to improve unidirectional RNNs. - The connection of row convolution is different form the 1D sequence + The connection of row convolution is different from the 1D sequence convolution. Assumed that, the future context-length is k, that is to say, it can get the output at timestep t by using the the input feature from t-th timestep to (t+k+1)-th timestep. Assumed that the hidden dim of input @@ -6243,7 +6289,7 @@ def row_conv_layer(input, :param act: Activation Type. Default is linear activation. :type act: BaseActivation :param param_attr: The Parameter Attribute. If None, the parameter will be - initialized smartly. It's better set it by yourself. + initialized smartly. It's better to set it by yourself. :type param_attr: ParameterAttribute :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -6290,7 +6336,7 @@ def prelu_layer(input, prelu = prelu_layer(input=layers, partial_sum=1) - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -6343,7 +6389,7 @@ def gated_unit_layer(input, The gated unit layer implements a simple gating mechanism over the input. The input :math:`X` is first projected into a new space :math:`X'`, and it is also used to produce a gate weight :math:`\sigma`. Element-wise - prodict between :match:`X'` and :math:`\sigma` is finally returned. + product between :match:`X'` and :math:`\sigma` is finally returned. Reference: Language Modeling with Gated Convolutional Networks @@ -6363,7 +6409,7 @@ def gated_unit_layer(input, :type size: int :param act: activation type of the projected input. :type act: BaseActivation - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param gate_attr: Attributes to tune the gate output, for example, error clipping threshold, dropout and so on. See ExtraLayerAttribute for @@ -6439,10 +6485,10 @@ def switch_order_layer(input, :param input: The input layer. :type input: LayerOutput - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring - :param reshape: reshape matrix by axises. - :type reshape: Dict + :param reshape_axis: Specify the axises of 'height'. Its value should be positive and less than 4. + :type reshape_axis: int :return: LayerOutput object. :rtype: LayerOutput """ @@ -6492,7 +6538,7 @@ def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None): :type partial_sum: int :param shape: The shape to be cropped. Default is None. :type shape: Sequence | None - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6538,7 +6584,7 @@ def sub_nested_seq_layer(input, selected_indices, name=None): :type input: LayerOutput :param selected_indices: a set of sequence indices in the nested sequence. :type input: LayerOutput - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6576,7 +6622,7 @@ def clip_layer(input, min, max, name=None): clip = clip_layer(input=input_layer, min=-10, max=10) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput. @@ -6621,7 +6667,7 @@ def seq_slice_layer(input, starts, ends, name=None): seq_silce = seq_slice_layer(input=input_seq, starts=start_pos, ends=end_pos) - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input for this layer, it should be a sequence. :type input: LayerOutput @@ -6675,12 +6721,12 @@ def kmax_seq_score_layer(input, name=None, beam_size=1): kmax_indices = kmax_seq_score_layer(input=input_layer, beam_size) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. It stores scores over a sequence or a nested sequence and its size must be 1. :type input: LayerOutput. - :param beam_size: squence indices with top beam_size scores are returned. + :param beam_size: sequence indices with top beam_size scores are returned. :type beam_size: double :return: LayerOutput object. :rtype: LayerOutput @@ -6733,7 +6779,7 @@ def img_conv3d_layer(input, bias_attr=False, act=ReluActivation()) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Layer Input. :type input: LayerOutput @@ -6752,7 +6798,7 @@ def img_conv3d_layer(input, :type padding: int|tuple|list :param bias_attr: Convolution bias attribute. None means default bias. False means no bias. - :type bias_attr: ParameterAttribute|False + :type bias_attr: ParameterAttribute|None|Bool|Any :param num_channels: number of input channels. If None will be set automatically from previous output. :type num_channels: int @@ -6864,14 +6910,17 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): scale_shift = scale_shift_layer(input=input_layer, bias_attr=False) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput. :param param_attr: The parameter attribute of scaling. :type param_attr: ParameterAttribute - :param bias_attr: The parameter attribute of shifting. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py new file mode 100644 index 0000000000000000000000000000000000000000..8f6d2be17758b7f6604d2db74fe466fb30695bd5 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -0,0 +1,223 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestExp(OpTest): + def setUp(self): + self.op_type = "exp" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.exp(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSigmoid(OpTest): + def setUp(self): + self.op_type = "sigmoid" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + +class TestTanh(OpTest): + def setUp(self): + self.op_type = "tanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.tanh(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSqrt(OpTest): + def setUp(self): + self.op_type = "sqrt" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.sqrt(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestAbs(OpTest): + def setUp(self): + self.op_type = "abs" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + # Because we set delta = 0.005 in caculating numeric gradient, + # if x is too small, such as 0.002, x_neg will be -0.003 + # x_pos will be 0.007, so the numeric gradient is unaccurate. + # we should avoid this + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.abs(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestRelu(OpTest): + def setUp(self): + self.op_type = "relu" + x = np.random.uniform(-1, 1, [11, 17]).astype("float32") + # The same reason with TestAbs + x[np.abs(x) < 0.005] = 0.02 + self.inputs = {'X': x} + self.outputs = {'Y': np.maximum(self.inputs['X'], 0)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestBRelu(OpTest): + def setUp(self): + self.op_type = "brelu" + x = np.random.uniform(-1, 1, [4, 4]).astype("float32") + t_min = 1 + t_max = 4 + # The same with TestAbs + x[np.abs(x - t_min) < 0.005] = t_min + 0.02 + x[np.abs(x - t_max) < 0.005] = t_max + 0.02 + + self.inputs = {'X': x} + self.attrs = {'t_min': t_min, 't_max': t_max} + t = np.copy(x) + t[t < t_min] = t_min + t[t > t_max] = t_max + self.outputs = {'Y': t} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSoftRelu(OpTest): + def setUp(self): + self.op_type = "soft_relu" + x = np.random.uniform(-3, 3, [4, 4]).astype("float32") + threshold = 2 + # The same reason with TestAbs + x[np.abs(x - threshold) < 0.005] = threshold + 0.02 + x[np.abs(x + threshold) < 0.005] = -threshold + 0.02 + self.inputs = {'X': x} + self.attrs = {'threshold': threshold} + t = np.copy(x) + t[t < -threshold] = -threshold + t[t > threshold] = threshold + self.outputs = {'Y': np.log((np.exp(t) + 1))} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestReciprocal(OpTest): + def setUp(self): + self.op_type = "reciprocal" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.outputs = {'Y': np.reciprocal(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.01) + + +class TestLog(OpTest): + def setUp(self): + self.op_type = "log" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.log(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestSquare(OpTest): + def setUp(self): + self.op_type = "square" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + self.outputs = {'Y': np.square(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +class TestPow(OpTest): + def setUp(self): + self.op_type = "pow" + self.inputs = {'X': np.random.uniform(1, 2, [11, 17]).astype("float32")} + self.attrs = {'factor': 3} + self.outputs = {'Y': np.power(self.inputs['X'], 3)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.02) + + +class TestSTanh(OpTest): + def setUp(self): + self.op_type = "stanh" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") + } + scale_a = 2.0 / 3.0 + scale_b = 1.7159 + self.attrs = {'scale_a': scale_a, 'scale_b': scale_b} + self.outputs = {'Y': scale_b * np.tanh(self.inputs['X'] * scale_a)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.007) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_clip_op.py b/python/paddle/v2/framework/tests/test_clip_op.py new file mode 100644 index 0000000000000000000000000000000000000000..5df6a494989017bab0416e0af962b2a85db046ba --- /dev/null +++ b/python/paddle/v2/framework/tests/test_clip_op.py @@ -0,0 +1,58 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestClipOp(OpTest): + def setUp(self): + self.max_relative_error = 0.006 + self.initTestCase() + input = np.random.random(self.shape).astype("float32") + input[np.abs(input - self.min) < self.max_relative_error] = 0.5 + input[np.abs(input - self.max) < self.max_relative_error] = 0.5 + self.op_type = "clip" + self.inputs = {'X': input, } + self.attrs = {} + self.attrs['min'] = self.min + self.attrs['max'] = self.max + self.outputs = { + 'Out': np.clip(self.inputs['X'], self.attrs['min'], + self.attrs['max']) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad( + ['X'], 'Out', max_relative_error=self.max_relative_error) + + def initTestCase(self): + self.shape = (4, 4) + self.max = 0.7 + self.min = 0.1 + + +class TestCase1(TestClipOp): + def initTestCase(self): + self.shape = (8, 16, 8) + self.max = 0.7 + self.min = 0 + + +class TestCase2(TestClipOp): + def initTestCase(self): + self.shape = (8, 16) + self.max = 1 + self.min = 0 + + +class TestCase3(TestClipOp): + def initTestCase(self): + self.shape = (4, 8, 16) + self.max = 0.7 + self.min = 0.2 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_conv2d_op.py b/python/paddle/v2/framework/tests/test_conv2d_op.py new file mode 100644 index 0000000000000000000000000000000000000000..118a5fc1cde5f4a908b065d581956e0855d50a52 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_conv2d_op.py @@ -0,0 +1,103 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestConv2dOp(OpTest): + def setUp(self): + self.init_groups() + self.op_type = "conv2d" + batch_size = 2 + input_channels = 3 + input_height = 5 + input_width = 5 + output_channels = 6 + filter_height = 3 + filter_width = 3 + stride = 1 + padding = 0 + output_height = (input_height - filter_height + 2 * padding + ) / stride + 1 + output_width = (input_width - filter_width + 2 * padding) / stride + 1 + input = np.random.random((batch_size, input_channels, input_height, + input_width)).astype("float32") + + filter = np.random.random( + (output_channels, input_channels / self.groups, filter_height, + filter_width)).astype("float32") + output = np.ndarray( + (batch_size, output_channels, output_height, output_width)) + + self.inputs = {'Input': input, 'Filter': filter} + self.attrs = { + 'strides': [1, 1], + 'paddings': [0, 0], + 'groups': self.groups + } + + output_group_channels = output_channels / self.groups + input_group_channels = input_channels / self.groups + for batchid in xrange(batch_size): + for group in xrange(self.groups): + for outchannelid in range(group * output_group_channels, + (group + 1) * output_group_channels): + for rowid in xrange(output_height): + for colid in xrange(output_width): + start_h = (rowid * stride) - padding + start_w = (colid * stride) - padding + output_value = 0.0 + for inchannelid in range( + group * input_group_channels, + (group + 1) * input_group_channels): + for frowid in xrange(filter_height): + for fcolid in xrange(filter_width): + input_value = 0.0 + inrowid = start_h + frowid + incolid = start_w + fcolid + if ((inrowid >= 0 and + inrowid < input_height) and + (incolid >= 0 and + incolid < input_width)): + input_value = input[batchid][ + inchannelid][inrowid][incolid] + filter_value = filter[outchannelid][ + inchannelid % input_group_channels][ + frowid][fcolid] + output_value += input_value * filter_value + output[batchid][outchannelid][rowid][ + colid] = output_value + + self.outputs = {'Output': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad( + set(['Input', 'Filter']), 'Output', max_relative_error=0.05) + + def test_check_grad_no_filter(self): + self.check_grad( + ['Input'], + 'Output', + max_relative_error=0.05, + no_grad_set=set(['Filter'])) + + def test_check_grad_no_input(self): + self.check_grad( + ['Filter'], + 'Output', + max_relative_error=0.05, + no_grad_set=set(['Input'])) + + def init_groups(self): + self.groups = 1 + + +class TestWithGroup(TestConv2dOp): + def init_groups(self): + self.groups = 3 + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_crop_op.py b/python/paddle/v2/framework/tests/test_crop_op.py new file mode 100644 index 0000000000000000000000000000000000000000..62c883bdc130021d06c33ded9c2865505da0b719 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_crop_op.py @@ -0,0 +1,91 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def crop(data, offsets, crop_shape): + def indexOf(shape, index): + result = [] + for dim in reversed(shape): + result.append(index % dim) + index = index / dim + return result[::-1] + + result = [] + for i, value in enumerate(data.flatten()): + index = indexOf(data.shape, i) + selected = True + if len(index) == len(offsets): + for j, offset in enumerate(offsets): + selected = selected and index[j] >= offset and index[ + j] < crop_shape[j] + offset + if selected: + result.append(value) + return np.array(result).reshape(crop_shape) + + +class TestCropOp(OpTest): + def setUp(self): + self.op_type = "crop" + self.crop_by_input = False + self.attrs = {} + self.initTestCase() + self.attrs['offsets'] = self.offsets + if self.crop_by_input: + self.inputs = { + 'X': np.random.random(self.x_shape).astype("float32"), + 'Y': np.random.random(self.crop_shape).astype("float32") + } + else: + self.attrs['shape'] = self.crop_shape + self.inputs = { + 'X': np.random.random(self.x_shape).astype("float32"), + } + self.outputs = { + 'Out': crop(self.inputs['X'], self.offsets, self.crop_shape) + } + + def initTestCase(self): + self.x_shape = (8, 8) + self.crop_shape = (2, 2) + self.offsets = [1, 2] + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X'], 'Out', max_relative_error=0.006) + + +class TestCase1(TestCropOp): + def initTestCase(self): + self.x_shape = (16, 8, 32) + self.crop_shape = [2, 2, 3] + self.offsets = [1, 5, 3] + + +class TestCase2(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 8) + self.crop_shape = [4, 8] + self.offsets = [0, 0] + + +class TestCase3(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 8, 16) + self.crop_shape = [2, 2, 3] + self.offsets = [1, 5, 3] + self.crop_by_input = True + + +class TestCase4(TestCropOp): + def initTestCase(self): + self.x_shape = (4, 4) + self.crop_shape = [4, 4] + self.offsets = [0, 0] + self.crop_by_input = True + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_cross_entropy_op.py new file mode 100644 index 0000000000000000000000000000000000000000..0206ca064be87afe204aa99021979b7ddc3c5d63 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_cross_entropy_op.py @@ -0,0 +1,89 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestCrossEntropyOp1(OpTest): + """Test standard cross-entropy, with index representation of labels. + """ + + def setUp(self): + self.op_type = "cross_entropy" + batch_size = 30 + class_num = 10 + X = np.random.uniform(0.1, 1.0, + [batch_size, class_num]).astype("float32") + label = np.random.randint(0, class_num, (batch_size, 1), dtype="int32") + cross_entropy = np.asmatrix( + [[-np.log(X[i][label[i][0]])] for i in range(X.shape[0])], + dtype="float32") + self.inputs = {"X": X, "Label": label} + self.outputs = {"Y": cross_entropy} + self.attrs = {'soft_label': 0} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["X"], "Y") + + +class TestCrossEntropyOp2(OpTest): + """Test soft-label cross-entropy, with vecterized soft labels. + """ + + def setUp(self): + self.op_type = "cross_entropy" + batch_size = 10 + class_num = 5 + X = np.random.uniform(0.1, 1.0, + [batch_size, class_num]).astype("float32") + label = np.random.uniform(0.1, 1.0, + [batch_size, class_num]).astype("float32") + label /= label.sum(axis=1, keepdims=True) + cross_entropy = (-label * np.log(X)).sum( + axis=1, keepdims=True).astype("float32") + self.inputs = {'X': X, 'Label': label} + self.outputs = {'Y': cross_entropy} + self.attrs = {'soft_label': 1} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y') + + +class TestCrossEntropyOp3(OpTest): + """Test one-hot cross-entropy, with vecterized one-hot representation of + labels. + """ + + def setUp(self): + self.op_type = "cross_entropy" + batch_size = 30 + class_num = 10 + X = np.random.uniform(0.1, 1.0, + [batch_size, class_num]).astype("float32") + label_index = np.random.randint( + 0, class_num, (batch_size), dtype="int32") + label = np.zeros(X.shape) + label[np.arange(batch_size), label_index] = 1 + cross_entropy = np.asmatrix( + [[-np.log(X[i][label_index[i]])] for i in range(X.shape[0])], + dtype="float32") + cross_entropy2 = (-label * np.log(X)).sum( + axis=1, keepdims=True).astype("float32") + self.inputs = {'X': X, 'Label': label} + self.outputs = {'Y': cross_entropy} + self.attrs = {'soft_label': 1} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_dropout_op.py b/python/paddle/v2/framework/tests/test_dropout_op.py new file mode 100644 index 0000000000000000000000000000000000000000..3638fee1a1c26195791bc1f5a46dd749da0aee95 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_dropout_op.py @@ -0,0 +1,59 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestDropoutOp(OpTest): + def setUp(self): + self.op_type = "dropout" + self.inputs = {'X': np.random.random((32, 64)).astype("float32")} + self.attrs = {'dropout_prob': 0.0, 'is_training': 1} + self.outputs = {'Out': self.inputs['X'], 'Mask': np.ones((32, 64))} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X'], 'Out', max_relative_error=0.05) + + +class TestDropoutOp2(TestDropoutOp): + def setUp(self): + self.op_type = "dropout" + self.inputs = {'X': np.random.random((32, 64)).astype("float32")} + self.attrs = {'dropout_prob': 1.0, 'is_training': 1} + self.outputs = {'Out': np.zeros((32, 64)), 'Mask': np.zeros((32, 64))} + + +class TestDropoutOp3(TestDropoutOp): + def setUp(self): + self.op_type = "dropout" + self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")} + self.attrs = {'dropout_prob': 0.0, 'is_training': 1} + self.outputs = {'Out': self.inputs['X'], 'Mask': np.ones((32, 64, 2))} + + +class TestDropoutOp4(OpTest): + def setUp(self): + self.op_type = "dropout" + self.inputs = {'X': np.random.random((32, 64)).astype("float32")} + self.attrs = {'dropout_prob': 0.35, 'is_training': 0} + self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} + + def test_check_output(self): + self.check_output() + + +class TestDropoutOp5(OpTest): + def setUp(self): + self.op_type = "dropout" + self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")} + self.attrs = {'dropout_prob': 0.75, 'is_training': 0} + self.outputs = {'Out': self.inputs['X'] * self.attrs['dropout_prob']} + + def test_check_output(self): + self.check_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_mnist.py b/python/paddle/v2/framework/tests/test_mnist.py index f6f8f49b797fb6e5016a5e309f12f192d5096431..66452cb3965d28fd15e814833079621410775c17 100644 --- a/python/paddle/v2/framework/tests/test_mnist.py +++ b/python/paddle/v2/framework/tests/test_mnist.py @@ -128,7 +128,7 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None): def cross_entropy_layer(net, input, label): cost_name = "cross_entropy_%d" % uniq_id() cross_entropy_op = Operator( - "onehot_cross_entropy", X=input, label=label, Y=cost_name) + "cross_entropy", X=input, Label=label, Y=cost_name) net.append_op(cross_entropy_op) scope.new_var(cost_name) net.infer_shape(scope) @@ -181,7 +181,7 @@ def error_rate(predict, label): images = data_layer(name="pixel", dims=[BATCH_SIZE, 784]) -labels = data_layer(name="label", dims=[BATCH_SIZE]) +labels = data_layer(name="label", dims=[BATCH_SIZE, 1]) fc1 = fc_layer(net=forward_net, input=images, size=100, act="sigmoid") fc2 = fc_layer(net=forward_net, input=fc1, size=100, act="sigmoid") predict = fc_layer(net=forward_net, input=fc2, size=10, act="softmax") @@ -215,6 +215,7 @@ def test(cost_name): for data in test_reader(): image_data = numpy.array(map(lambda x: x[0], data)).astype("float32") label_data = numpy.array(map(lambda x: x[1], data)).astype("int32") + label_data = numpy.expand_dims(label_data, axis=1) feed_data(images, image_data) feed_data(labels, label_data) @@ -235,6 +236,7 @@ for pass_id in range(PASS_NUM): for data in train_reader(): image_data = numpy.array(map(lambda x: x[0], data)).astype("float32") label_data = numpy.array(map(lambda x: x[1], data)).astype("int32") + label_data = numpy.expand_dims(label_data, axis=1) feed_data(images, image_data) feed_data(labels, label_data) diff --git a/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..a7e2b57529b0723b4ab18b73801cd2816d8025dd --- /dev/null +++ b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py @@ -0,0 +1,39 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def modified_huber_loss_forward(val): + if val < -1: + return -4 * val + elif val < 1: + return (1 - val) * (1 - val) + else: + return 0 + + +class TestModifiedHuberLossOp(OpTest): + def setUp(self): + self.op_type = 'modified_huber_loss' + samples_num = 32 + self.inputs = { + 'X': np.random.uniform(-1, 1., (samples_num, 1)).astype('float32'), + 'Y': np.random.choice([0, 1], samples_num).reshape((samples_num, 1)) + } + product_res = self.inputs['X'] * (2 * self.inputs['Y'] - 1) + loss = np.vectorize(modified_huber_loss_forward)(product_res) + + self.outputs = { + 'IntermediateVal': product_res, + 'Out': loss.reshape((samples_num, 1)) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out', max_relative_error=0.005) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_onehot_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_onehot_cross_entropy_op.py deleted file mode 100644 index fd3cbdb80374865ccf113768856096bf49dce643..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_onehot_cross_entropy_op.py +++ /dev/null @@ -1,30 +0,0 @@ -import unittest -import numpy -from op_test import OpTest - - -class TestOnehotCrossEntropyOp(OpTest): - def setUp(self): - self.op_type = "onehot_cross_entropy" - batch_size = 30 - class_num = 10 - - X = numpy.random.uniform(0.1, 1.0, - [batch_size, class_num]).astype("float32") - labels = numpy.random.randint(0, class_num, batch_size, dtype="int32") - - cross_entropy = numpy.asmatrix( - [[-numpy.log(X[i][labels[i]])] for i in range(X.shape[0])], - dtype="float32") - self.inputs = {"X": X, "label": labels} - self.outputs = {"Y": cross_entropy} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Y") - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_prelu_op.py b/python/paddle/v2/framework/tests/test_prelu_op.py new file mode 100644 index 0000000000000000000000000000000000000000..2b6b7db36808a4b68c55328a1eb9ac212c18b678 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_prelu_op.py @@ -0,0 +1,28 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class PReluTest(OpTest): + def setUp(self): + self.op_type = "prelu" + x_np = np.random.normal(size=(10, 10)).astype("float32") + x_np_sign = np.sign(x_np) + x_np = x_np_sign * np.maximum(x_np, .005) + alpha_np = np.array([.1]) + self.inputs = {'X': x_np, 'Alpha': alpha_np} + out_np = np.maximum(self.inputs['X'], 0.) + out_np = out_np + np.minimum(self.inputs['X'], + 0.) * self.inputs['Alpha'] + assert out_np is not self.inputs['X'] + self.outputs = {'Out': out_np} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_rank_loss_op.py b/python/paddle/v2/framework/tests/test_rank_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..0e41ab1b3fd8fa8b62c5f3b914b752918119a265 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_rank_loss_op.py @@ -0,0 +1,32 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestRankLossOp(OpTest): + def setUp(self): + self.op_type = "rank_loss" + batch_size = 5 + # labels_{i} = {0, 1.0} or {0, 0.5, 1.0} + label = np.random.randint(0, 2, size=(batch_size, 1)).astype("float32") + left = np.random.random((batch_size, 1)).astype("float32") + right = np.random.random((batch_size, 1)).astype("float32") + loss = np.log(1.0 + np.exp(left - right)) - label * (left - right) + self.inputs = {'Label': label, 'Left': left, 'Right': right} + self.outputs = {'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(["Left", "Right"], "Out") + + def test_check_grad_ignore_left(self): + self.check_grad(["Right"], "Out", no_grad_set=set('Left')) + + def test_check_grad_ignore_right(self): + self.check_grad(["Left"], "Out", no_grad_set=set('Right')) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 22e680fd783ec681e95326fb84db34570265cffc..79eda70021b76cd06e4c40740b1ca49476f4c503 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -59,7 +59,6 @@ class PySimpleRNNTest(unittest.TestCase): def test_forward(self): output = self.rnn.forward() - print 'output', output def create_tensor(scope, name, shape, np_data): @@ -103,7 +102,7 @@ class TestRecurrentOp(unittest.TestCase): ctx = core.DeviceContext.create(core.CPUPlace()) self.rnnop.infer_shape(self.scope) self.rnnop.run(self.scope, ctx) - return np.array(self.scope.find_var("h").get_tensor()) + return np.array(self.scope.find_var("h@mem").get_tensor()) def create_global_variables(self): # create inlink @@ -123,8 +122,7 @@ class TestRecurrentOp(unittest.TestCase): create_tensor(self.scope, "h_boot", [self.batch_size, self.input_dim], h_boot_np_data) self.scope.new_var("step_scopes") - self.scope.new_var("h@alias") - self.scope.new_var("h") + self.scope.new_var("h@mem") def create_rnn_op(self): # create RNNOp @@ -134,20 +132,18 @@ class TestRecurrentOp(unittest.TestCase): boot_memories=["h_boot"], step_net="stepnet", # outputs - outlinks=["h"], + outlinks=["h@mem"], step_scopes="step_scopes", # attributes - inlink_alias=["x@alias"], - outlink_alias=["h@alias"], pre_memories=["h@pre"], - memories=["h@alias"]) + memories=["h@mem"]) def create_step_net(self): stepnet = core.Net.create() - x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") + x_fc_op = Operator("mul", X="x", Y="W", Out="Wx") h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") sum_op = Operator("add", X="Wx", Y="Uh", Out="sum") - sig_op = Operator("sigmoid", X="sum", Y="h@alias") + sig_op = Operator("sigmoid", X="sum", Y="h@mem") for op in [x_fc_op, h_fc_op, sum_op, sig_op]: stepnet.append_op(op) diff --git a/python/paddle/v2/framework/tests/test_sigmoid_op.py b/python/paddle/v2/framework/tests/test_sigmoid_op.py deleted file mode 100644 index d65d887db4af58c40e4e78fdbfd8e8ee668b7ee3..0000000000000000000000000000000000000000 --- a/python/paddle/v2/framework/tests/test_sigmoid_op.py +++ /dev/null @@ -1,22 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestSigmoidOp(OpTest): - def setUp(self): - self.op_type = "sigmoid" - self.inputs = { - 'X': np.random.uniform(0.1, 1, [11, 17]).astype("float32") - } - self.outputs = {'Y': 1 / (1 + np.exp(-self.inputs['X']))} - - def test_check_output(self): - self.check_output() - - def test_check_grad(self): - self.check_grad(["X"], "Y", max_relative_error=0.007) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py new file mode 100644 index 0000000000000000000000000000000000000000..be940327ec910ccb9de59d45029513ff4779443b --- /dev/null +++ b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py @@ -0,0 +1,87 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def smooth_l1_loss_forward(val, sigma2): + abs_val = abs(val) + if abs_val < 1.0 / sigma2: + return 0.5 * val * val * sigma2 + else: + return abs_val - 0.5 / sigma2 + + +class TestSmoothL1LossOp1(OpTest): + def setUp(self): + self.op_type = "smooth_l1_loss" + dims = (5, 10) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2).sum(1) + loss = loss.reshape((dims[0], 1)) + self.outputs = {'Diff': diff, 'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.02) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.03, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.03, no_grad_set=set('Y')) + + +class TestSmoothL1LossOp2(OpTest): + def setUp(self): + self.op_type = "smooth_l1_loss" + dims = (5, 10) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32"), + 'InsideWeight': np.random.random(dims).astype("float32"), + 'OutsideWeight': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + diff = diff * self.inputs['InsideWeight'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2) + loss = loss * self.inputs['OutsideWeight'] + loss = loss.sum(1).reshape((dims[0], 1)) + self.outputs = {'Diff': diff, 'Out': loss} + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.03) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight'])) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], + 'Out', + max_relative_error=0.03, + no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_transpose_op.py b/python/paddle/v2/framework/tests/test_transpose_op.py new file mode 100644 index 0000000000000000000000000000000000000000..9409cbaa00f792b60d5950556b869108aa732478 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_transpose_op.py @@ -0,0 +1,56 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestTransposeOp(OpTest): + def setUp(self): + self.initTestCase() + self.op_type = "transpose" + self.inputs = {'X': np.random.random(self.shape).astype("float32")} + self.attrs = {'axis': list(self.axis)} + self.outputs = {'Out': self.inputs['X'].transpose(self.axis)} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + def initTestCase(self): + self.shape = (3, 4) + self.axis = (1, 0) + + +class TestCase0(TestTransposeOp): + def initTestCase(self): + self.shape = (3, ) + self.axis = (0, ) + + +class TestCase1(TestTransposeOp): + def initTestCase(self): + self.shape = (3, 4, 5) + self.axis = (0, 2, 1) + + +class TestCase2(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5) + self.axis = (0, 2, 3, 1) + + +class TestCase3(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5, 6) + self.axis = (4, 2, 3, 1, 0) + + +class TestCase4(TestTransposeOp): + def initTestCase(self): + self.shape = (2, 3, 4, 5, 6, 1) + self.axis = (4, 2, 3, 1, 0, 5) + + +if __name__ == '__main__': + unittest.main()