diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cd8eb12f6b23c67e8fb22f43d57afd4a96770fd..61b989dc698798eca932516e558c63f107ef2754 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,3 +21,10 @@ sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29 hooks: - id: clang-formater +- repo: https://github.com/dnephin/pre-commit-golang + sha: e4693a4c282b4fc878eda172a929f7a6508e7d16 + hooks: + - id: go-fmt + files: (.*\.go) + - id: go-lint + files: (.*\.go) diff --git a/.travis.yml b/.travis.yml index a53bd1809416d6f14a1ec7f603622d3303d1ab28..498674469b27f585af798b95f30b74ebed99e32c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,16 +33,17 @@ addons: - ccache before_install: - if [[ "$JOB" == "check_style" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python + # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker - pip install rarfile + - curl https://glide.sh/get | bash - eval "$(GIMME_GO_VERSION=1.8.3 gimme)" - | function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } script: - | - timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout + timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi; notifications: email: diff --git a/AUTHORS.md b/AUTHORS.md index d5baee2161aa1d5360056e03ca67d5b2fe9ff7d2..4db4a4a8e7441b07ce2db4adff043bb99a09014b 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,28 +1,48 @@ | Github account | name | |---|---| -| reyoung | Yang Yu | +| backyes | Yan-Fei Wang | +| beckett1124 | Bin Qi | +| Canpio | Jia-Yi Feng | +| chengxiaohua1105 | Xiao-Hua Cheng | +| cxwangyi, yiwangbaidu, wangkuiyi | Yi Wang | +| cxysteven | Xing-Yi Cheng | +| dzhwinter | Zhi-Hong Dong | +| emailweixu | Wei Xu | | gangliao | Gang Liao | -| luotao01 | Tao Luo | -| jacquesqiao | Long-Fei Qiao | -| qingqing01 | Qing-Qing Dang | +| gongweibao | Wei-Bao Gong | +| Guo Sheng | Sheng Guo | +| Haichao-Zhang | Hai-Chao Zhang | | hedaoyuan | Dao-Yuan He | -| wangyang59 | Yang Wang | +| helinwang | He-Lin Wang | +| jacquesqiao | Long-Fei Qiao | +| kuke | Yi-Bing Liu | +| lcy-seso | Ying Cao | +| lipeng-unisound | Peng Li | +| liuyuan | Yuan Liu | +| livc | Zhao Li | +| llxxxll | Yong-Feng Liu | +| luotao01 | Tao Luo | +| lzhao4ever | Liang Zhao | +| NHZlX | Zhao-Long Xing | +| pakchoi | Chuan-Jiang Song | +| pengli09 | Peng Li | +| pkuyym | Ya-Ming Yang | | QiJune | Jun Qi | +| qingqing01 | Qing-Qing Dang | +| reyoung | Yang Yu | +| Superjom | Chun-Wei Yan | | tianbingsz | Tian-Bing Xu | -| cxwangyi, yiwangbaidu, wangkuiyi | Yi Wang | | typhoonzero | Yi Wu | -| backyes | Yan-Fei Wang | -| pengli09 | Peng Li | -| livc | Zhao Li | +| wanghaoshuang | Hao-Shuang Wang | +| wangyang59 | Yang Wang | +| wangzhen-nlp | Zhen Wang | +| wen-bo-yang | Wen-Bo Yang | +| wwhu | Wei-Wei Hu | +| xinghai-sun | Xing-Hai Sun | | Xreki | Yi-Qun Liu | +| xujun05 | Jun Xu | +| xushaoyong | Shao-Yong Xu | | Yancey1989 | Xu Yan | -| emailweixu | Wei Xu | -| wen-bo-yang | Wen-Bo Yang | -| helinwang | He-Lin Wang | -| lcy-seso | Ying Cao | -| Zrachel | Rui-Qing Zhang | -| Haichao-Zhang | Hai-Chao Zhang | -| gongweibao | Wei-Bao Gong | -| lzhao4ever | Liang Zhao | +| zhaopu7 | Pu Zhao | | zhouxiao-coder | Xiao Zhou | -| lipeng-unisound | Peng Li | +| Zrachel | Rui-Qing Zhang | diff --git a/CMakeLists.txt b/CMakeLists.txt index ed3c390066dfac2322d802c6039bc7155a36e38a..5bedbbefa85a730ff2934a12597988a67e73c1a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,7 @@ option(COVERALLS_UPLOAD "Package code coverage data to coveralls" OFF) option(ON_TRAVIS "Exclude special unit test on Travis CI" OFF) option(WITH_C_API "Compile PaddlePaddle with C-API(Prediction)" OFF) option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF) +option(GLIDE_INSTALL "Download and install go dependencies " ON) option(USE_NNPACK "Compile PaddlePaddle with NNPACK library" OFF) # CMAKE_BUILD_TYPE @@ -96,6 +97,8 @@ include(external/warpctc) # download, build, install warpctc include(external/any) # download libn::any include(external/eigen) # download eigen3 +include(cudnn) # set cudnn libraries, must before configure +include(configure) # add paddle env configuration include(generic) # simplify cmake module include(package) # set paddle packages include(cpplint) # set paddle c++ style @@ -103,15 +106,14 @@ include(ccache) # set ccache for compilation include(util) # set unittest and link libs include(rdma) # set rdma libraries include(flags) # set paddle compile flags -include(cudnn) # set cudnn libraries include(version) # set PADDLE_VERSION include(coveralls) # set code coverage -include(configure) # add paddle env configuration + include_directories("${PROJ_ROOT}") include_directories("${PROJ_ROOT}/paddle/cuda/include") include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto") -include_directories("${CMAKE_CURRENT_BINARY_DIR}/go/pserver/cclient") +include_directories("${CMAKE_CURRENT_BINARY_DIR}/go/pserver/client/c") include_directories(${Boost_INCLUDE_DIRS}) set(EXTERNAL_LIBS @@ -139,8 +141,7 @@ add_subdirectory(proto) # "add_subdirectory(paddle)" and "add_subdirectory(python)" should be # placed after this block, because they depends on it. if(WITH_GOLANG) - add_subdirectory(go/master/c) - add_subdirectory(go/pserver/cclient) + add_subdirectory(go) endif(WITH_GOLANG) add_subdirectory(paddle) diff --git a/Dockerfile b/Dockerfile index bf227737c5a67b006ccc221235daf6d8ad7b3bd8..ed5910d93b41dba8d50b2ba01c59c635797edd29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,14 +34,18 @@ RUN apt-get update && \ net-tools && \ apt-get clean -y -# Install Go +# Install Go and glide RUN wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \ tar -C /usr/local -xzf go.tgz && \ mkdir /root/gopath && \ + mkdir /root/gopath/bin && \ + mkdir /root/gopath/src && \ rm go.tgz ENV GOROOT=/usr/local/go GOPATH=/root/gopath # should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT. -ENV PATH=${PATH}:${GOROOT}/bin +ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin +# install glide +RUN curl -q https://glide.sh/get | sh # git credential to skip password typing RUN git config --global credential.helper store @@ -57,7 +61,7 @@ RUN pip install --upgrade pip && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ - pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ pip install rarfile # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use diff --git a/cmake/configure.cmake b/cmake/configure.cmake index f6dca6d575956fcb893f8e11970f84a739832d58..a4f98ec7d4af652d0dd0650f4906696ff3a4efb9 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -1,11 +1,11 @@ # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,6 +79,9 @@ if(WITH_GOLANG) set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go") file(MAKE_DIRECTORY ${GOPATH}) set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle") + file(MAKE_DIRECTORY "${PADDLE_IN_GOPATH}") + set(PADDLE_GO_PATH "${CMAKE_SOURCE_DIR}/go") + add_custom_target(go_path) add_custom_command(TARGET go_path # Symlink Paddle directory into GOPATH @@ -89,7 +92,22 @@ if(WITH_GOLANG) # We can't run `go get -d ./...` for every target, because # multiple `go get` can not run concurrently, but make need to be # able to run with multiple jobs. - COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./go/... WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + + if (GLIDE_INSTALL) + if(EXISTS $ENV{GOPATH}/bin/glide) + set(GLIDE "$ENV{GOPATH}/bin/glide") + else() + message(FATAL_ERROR "no glide executeble found: $ENV{GOPATH}/bin/glide") + endif() + + add_custom_target(go_vendor) + add_custom_command(TARGET go_vendor + COMMAND env GOPATH=${GOPATH} ${GLIDE} install + WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go" + ) + add_dependencies(go_vendor go_path) + endif() + endif(WITH_GOLANG) diff --git a/cmake/cpplint.cmake b/cmake/cpplint.cmake index 48f705818b70c92adef107fd3c973ae1ab3d34bb..6bbcd730e1b5ac49415cac676352e6df00eb6eb5 100644 --- a/cmake/cpplint.cmake +++ b/cmake/cpplint.cmake @@ -25,6 +25,7 @@ set(STYLE_FILTER "${STYLE_FILTER}-readability/casting") set(IGNORE_PATTERN .*ImportanceSampler.* .*cblas\\.h.* + .*\\.pb\\.txt .*LtrDataProvider.* .*MultiDataProvider.*) diff --git a/cmake/external/any.cmake b/cmake/external/any.cmake index 62eea42692b4191e53d0bbb0805786fd15ac7944..45e3764e8482a4cfc8ee72fe4d79f04a3c9b74fa 100644 --- a/cmake/external/any.cmake +++ b/cmake/external/any.cmake @@ -2,10 +2,10 @@ INCLUDE(ExternalProject) SET(ANY_SOURCE_DIR ${THIRD_PARTY_PATH}/any) -INCLUDE_DIRECTORIES(${ANY_SOURCE_DIR}/src/linb_any) +INCLUDE_DIRECTORIES(${ANY_SOURCE_DIR}/src/extern_lib_any) ExternalProject_Add( - linb_any + extern_lib_any ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/thelink2012/any.git" GIT_TAG "8fef1e93710a0edf8d7658999e284a1142c4c020" @@ -17,5 +17,15 @@ ExternalProject_Add( TEST_COMMAND "" ) +if (${CMAKE_VERSION} VERSION_LESS "3.3.0") + set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/lib_any_dummy.c) + file(WRITE ${dummyfile} "const char * dummy_any = \"${dummyfile}\";") + add_library(lib_any STATIC ${dummyfile}) +else() + add_library(lib_any INTERFACE) +endif() + +add_dependencies(lib_any extern_lib_any) + add_definitions(-DANY_IMPL_ANY_CAST_MOVEABLE) -LIST(APPEND external_project_dependencies linb_any) \ No newline at end of file +LIST(APPEND external_project_dependencies lib_any) diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake index 45f44f617dcb46062355df4e35d537086215a46d..3e6cedbb0d718cfd4454f95dedf7e02a24f2981b 100644 --- a/cmake/external/eigen.cmake +++ b/cmake/external/eigen.cmake @@ -2,10 +2,10 @@ INCLUDE(ExternalProject) SET(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3) -INCLUDE_DIRECTORIES(${EIGEN_SOURCE_DIR}/src/eigen3) +INCLUDE_DIRECTORIES(${EIGEN_SOURCE_DIR}/src/extern_eigen3) ExternalProject_Add( - eigen3 + extern_eigen3 ${EXTERNAL_PROJECT_LOG_ARGS} # for latest version, please get from official website # URL "https://bitbucket.org/eigen/eigen/get/3.3.4.tar.gz" @@ -26,4 +26,14 @@ ExternalProject_Add( TEST_COMMAND "" ) +if (${CMAKE_VERSION} VERSION_LESS "3.3.0") + set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/eigen3_dummy.c) + file(WRITE ${dummyfile} "const char * dummy_eigen3 = \"${dummyfile}\";") + add_library(eigen3 STATIC ${dummyfile}) +else() + add_library(eigen3 INTERFACE) +endif() + +add_dependencies(eigen3 extern_eigen3) + LIST(APPEND external_project_dependencies eigen3) diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index b70e94a170f17cc61f61673609e6eb941662ea62..bd401faa6eb8a583bce542db68852f8571681daf 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -38,12 +38,14 @@ ExternalProject_Add( CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${GLOG_INSTALL_DIR}/lib CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DWITH_GFLAGS=ON CMAKE_ARGS -Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags CMAKE_ARGS -DBUILD_TESTING=OFF CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=Release ) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 3c74944bc21a131fe90e61777d3dce8b3f21900a..e629d61585c2d2ff916187ee28d4fd089a5bd857 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -17,6 +17,65 @@ INCLUDE(ExternalProject) FIND_PACKAGE(Protobuf QUIET) SET(PROTOBUF_FOUND "OFF") +if(NOT COMMAND protobuf_generate_python) # before cmake 3.4, protobuf_genrerate_python is not defined. + function(protobuf_generate_python SRCS) + # shameless copy from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_PYTHON() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH) + get_filename_component(FIL_DIR ${FIL} DIRECTORY) + if(FIL_DIR) + set(FIL_WE "${FIL_DIR}/${FIL_WE}") + endif() + endif() + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py") + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py" + COMMAND ${Protobuf_PROTOC_EXECUTABLE} --python_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} + COMMENT "Running Python protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + endfunction() +endif() # Print and set the protobuf library information, # finish this cmake process and exit from this file. diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 88be13b2ac95172d5d9099d62a40449c6a01e98a..8f65decda77946878daffda0afffab8e0b74b83d 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -17,7 +17,7 @@ # generic.cmake defines CMakes functions that look like Bazel's # building rules (https://bazel.build/). # -# +# # ------------------------------------------- # C++ CUDA C++ Go # ------------------------------------------- @@ -25,51 +25,51 @@ # cc_binary nv_binary go_binary # cc_test nv_test go_test # ------------------------------------------- -# +# # To build a static library example.a from example.cc using the system # compiler (like GCC): -# +# # cc_library(example SRCS example.cc) -# +# # To build a static library example.a from multiple source files # example{1,2,3}.cc: -# +# # cc_library(example SRCS example1.cc example2.cc example3.cc) -# +# # To build a shared library example.so from example.cc: -# +# # cc_library(example SHARED SRCS example.cc) -# +# # To build a library using Nvidia's NVCC from .cu file(s), use the nv_ # prefixed version: -# +# # nv_library(example SRCS example.cu) -# +# # To specify that a library new_example.a depends on other libraies: -# +# # cc_library(new_example SRCS new_example.cc DEPS example) -# +# # Static libraries can be composed of other static libraries: -# +# # cc_library(composed DEPS dependent1 dependent2 dependent3) -# +# # To build an executable binary file from some source files and # dependent libraries: -# +# # cc_binary(example SRCS main.cc something.cc DEPS example1 example2) -# +# # To build an executable binary file using NVCC, use the nv_ prefixed # version: -# +# # nv_binary(example SRCS main.cc something.cu DEPS example1 example2) -# +# # To build a unit test binary, which is an executable binary with # GoogleTest linked: -# +# # cc_test(example_test SRCS example_test.cc DEPS example) -# +# # To build a unit test binary using NVCC, use the nv_ prefixed version: -# +# # nv_test(example_test SRCS example_test.cu DEPS example) # # It is pretty often that executable and test binaries depend on @@ -99,25 +99,44 @@ function(merge_static_libs TARGET_NAME) set(libs ${ARGN}) list(REMOVE_DUPLICATES libs) - # First get the file names of the libraries to be merged + # Get all propagation dependencies from the merged libraries foreach(lib ${libs}) - set(libfiles ${libfiles} $) + list(APPEND libs_deps ${${lib}_LIB_DEPENDS}) endforeach() if(APPLE) # Use OSX's libtool to merge archives + # To produce a library we need at least one source file. + # It is created by add_custom_command below and will helps + # also help to track dependencies. set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_dummy.c) + + # Make the generated dummy source file depended on all static input + # libs. If input lib changes,the source file is touched + # which causes the desired effect (relink). + add_custom_command(OUTPUT ${dummyfile} + COMMAND ${CMAKE_COMMAND} -E touch ${dummyfile} + DEPENDS ${libs}) + + # Generate dummy staic lib file(WRITE ${dummyfile} "const char * dummy = \"${dummyfile}\";") add_library(${TARGET_NAME} STATIC ${dummyfile}) + target_link_libraries(${TARGET_NAME} ${libs_deps}) + + foreach(lib ${libs}) + # Get the file names of the libraries to be merged + set(libfiles ${libfiles} $) + endforeach() add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles}) - else() # general UNIX: use "ar" to extract objects and re-add to a common lib + else() # general UNIX: use "ar" to extract objects and re-add to a common lib foreach(lib ${libs}) set(objlistfile ${lib}.objlist) # list of objects in the input library set(objdir ${lib}.objdir) add_custom_command(OUTPUT ${objdir} - COMMAND ${CMAKE_COMMAND} -E make_directory ${objdir}) + COMMAND ${CMAKE_COMMAND} -E make_directory ${objdir} + DEPENDS ${lib}) add_custom_command(OUTPUT ${objlistfile} COMMAND ${CMAKE_AR} -x "$" @@ -125,27 +144,27 @@ function(merge_static_libs TARGET_NAME) DEPENDS ${lib} ${objdir} WORKING_DIRECTORY ${objdir}) - # Empty dummy source file that goes into merged library - set(mergebase ${lib}.mergebase.c) - add_custom_command(OUTPUT ${mergebase} - COMMAND ${CMAKE_COMMAND} -E touch ${mergebase} - DEPENDS ${objlistfile}) + # Empty dummy source file that goes into merged library + set(mergebase ${lib}.mergebase.c) + add_custom_command(OUTPUT ${mergebase} + COMMAND ${CMAKE_COMMAND} -E touch ${mergebase} + DEPENDS ${objlistfile}) list(APPEND mergebases "${mergebase}") endforeach() - # We need a target for the output merged library add_library(${TARGET_NAME} STATIC ${mergebases}) + target_link_libraries(${TARGET_NAME} ${libs_deps}) + + # Get the file name of the generated library set(outlibfile "$") foreach(lib ${libs}) add_custom_command(TARGET ${TARGET_NAME} POST_BUILD - COMMAND ${CMAKE_AR} ru ${outlibfile} @"../${lib}.objlist" - WORKING_DIRECTORY ${lib}.objdir) + COMMAND ${CMAKE_AR} cr ${outlibfile} *.o + COMMAND ${CMAKE_RANLIB} ${outlibfile} + WORKING_DIRECTORY ${lib}.objdir) endforeach() - - add_custom_command(TARGET ${TARGET_NAME} POST_BUILD - COMMAND ${CMAKE_RANLIB} ${outlibfile}) endif() endfunction(merge_static_libs) @@ -162,6 +181,7 @@ function(cc_library TARGET_NAME) endif() if (cc_library_DEPS) add_dependencies(${TARGET_NAME} ${cc_library_DEPS}) + target_link_libraries(${TARGET_NAME} ${cc_library_DEPS}) endif() else(cc_library_SRCS) if (cc_library_DEPS) @@ -193,7 +213,7 @@ function(cc_test TARGET_NAME) add_executable(${TARGET_NAME} ${cc_test_SRCS}) target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main) add_dependencies(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main) - add_test(${TARGET_NAME} ${TARGET_NAME}) + add_test(NAME ${TARGET_NAME} COMMAND ${TARGET_NAME} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() endfunction(cc_test) @@ -211,6 +231,7 @@ function(nv_library TARGET_NAME) endif() if (nv_library_DEPS) add_dependencies(${TARGET_NAME} ${nv_library_DEPS}) + target_link_libraries(${TARGET_NAME} ${nv_library_DEPS}) endif() else(nv_library_SRCS) if (nv_library_DEPS) @@ -278,14 +299,17 @@ function(go_library TARGET_NAME) set(${TARGET_NAME}_LIB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${${TARGET_NAME}_LIB_NAME}" CACHE STRING "output library path for target ${TARGET_NAME}") file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") + string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + # FIXME: link path add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND rm "${${TARGET_NAME}_LIB_PATH}" # Golang build source code - COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} + COMMAND GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} -o "${${TARGET_NAME}_LIB_PATH}" - ${GO_SOURCE} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_dependencies(${TARGET_NAME} go_path) + "./${CMAKE_CURRENT_SOURCE_REL_DIR}/${GO_SOURCE}" + # must run under GOPATH + WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") + add_dependencies(${TARGET_NAME} go_vendor) endfunction(go_library) function(go_binary TARGET_NAME) @@ -293,12 +317,17 @@ function(go_binary TARGET_NAME) set(oneValueArgs "") set(multiValueArgs SRCS DEPS) cmake_parse_arguments(go_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + + # FIXME: link path add_custom_command(OUTPUT ${TARGET_NAME}_timestamp - COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build + COMMAND env LIBRARY_PATH=${CMAKE_BINARY_DIR}/go/pserver/client/c/:$ENV{LIBRARY_PATH} + GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}" - ${go_library_SRCS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS}) + "./${CMAKE_CURRENT_SOURCE_REL_DIR}/${go_binary_SRCS}" + WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") + # TODO: don't know what ${TARGET_NAME}_link does + add_custom_target(${TARGET_NAME} ALL DEPENDS go_vendor ${TARGET_NAME}_timestamp ${go_binary_DEPS}) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin) endfunction(go_binary) @@ -318,10 +347,19 @@ endfunction(go_test) function(proto_library TARGET_NAME) set(oneValueArgs "") - set(multiValueArgs SRCS) + set(multiValueArgs SRCS DEPS) cmake_parse_arguments(proto_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) set(proto_srcs) set(proto_hdrs) protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS}) - cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS protobuf) + cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS ${proto_library_DEPS} protobuf) endfunction() + +function(py_proto_compile TARGET_NAME) + set(oneValueArgs "") + set(multiValueArgs SRCS) + cmake_parse_arguments(py_proto_compile "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(py_srcs) + protobuf_generate_python(py_srcs ${py_proto_compile_SRCS}) + add_custom_target(${TARGET_NAME} ALL DEPENDS ${py_srcs}) +endfunction() \ No newline at end of file diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index c7b017bc07b25bc606fd838a5fb9d3715f4faecb..4f4a9187bcbe8ef902e923622552909808b121d6 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -445,6 +445,11 @@ smooth_l1_cost .. autoclass:: paddle.v2.layer.smooth_l1_cost :noindex: +multibox_loss +-------------- +.. autoclass:: paddle.v2.layer.multibox_loss + :noindex: + Check Layer ============ @@ -468,3 +473,11 @@ prelu -------- .. autoclass:: paddle.v2.layer.prelu :noindex: + +Detection output Layer +====================== + +detection_output +---------------- +.. autoclass:: paddle.v2.layer.detection_output + :noindex: diff --git a/doc_theme/templates/layout.html b/doc_theme/templates/layout.html index 65e61c5f298e19adc6330c378779a6edf418752e..9fca69dc4e7f0827acfc755a97a662350214b90e 100644 --- a/doc_theme/templates/layout.html +++ b/doc_theme/templates/layout.html @@ -101,7 +101,7 @@
- Folk me on Github + Fork me on Github
English diff --git a/go/.gitignore b/go/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..000e1fd55b63b8e532308b787c2708a6c3e5ac87 --- /dev/null +++ b/go/.gitignore @@ -0,0 +1,2 @@ +vendor/ +.glide/ diff --git a/go/CMakeLists.txt b/go/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..f00c70a0589a4f41a23164a95d505d4310d9157b --- /dev/null +++ b/go/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_subdirectory(pserver/client/c) +add_subdirectory(cmd/pserver) +add_subdirectory(cmd/master) +add_subdirectory(master/c) diff --git a/go/cmd/master/CMakeLists.txt b/go/cmd/master/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1058ffa86b3f00b5e9525edca39a843da9b62db8 --- /dev/null +++ b/go/cmd/master/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +go_binary(master SRC master.go DEPS paddle_go_optimizer) diff --git a/go/cmd/pserver/CMakeLists.txt b/go/cmd/pserver/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..51db6dff043db362460dcbb7328c99d3cdd51a9b --- /dev/null +++ b/go/cmd/pserver/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +go_binary(pserver SRCS pserver.go DEPS paddle_go_optimizer) diff --git a/go/cmd/pserver/pserver.go b/go/cmd/pserver/pserver.go index 8a42d4f8af1713e246f9efaf5dc7ba878c3b271e..31ef450f032f756fb32a0444a7e94a18ec2918a0 100644 --- a/go/cmd/pserver/pserver.go +++ b/go/cmd/pserver/pserver.go @@ -15,6 +15,7 @@ import ( func main() { port := flag.Int("port", 0, "port of the pserver") + index := flag.Int("index", -1, "index of this pserver, should be larger or equal than 0") etcdEndpoint := flag.String("etcd-endpoint", "http://127.0.0.1:2379", "comma separated endpoint string for pserver to connect to etcd") etcdTimeout := flag.Int("etcd-timeout", 5, "timeout for etcd calls") @@ -29,11 +30,16 @@ func main() { } log.SetLevel(level) - timeout := time.Second * time.Duration((*etcdTimeout)) - e := pserver.NewEtcdClient(*etcdEndpoint, *numPservers, timeout) - idx, err := e.Register() - if err != nil { - panic(err) + var idx int + if *index >= 0 { + idx = *index + } else { + timeout := time.Second * time.Duration((*etcdTimeout)) + e := pserver.NewEtcdClient(*etcdEndpoint, *numPservers, timeout) + idx, err = e.Register() + if err != nil { + panic(err) + } } s, err := pserver.NewService(idx) diff --git a/go/glide.lock b/go/glide.lock new file mode 100644 index 0000000000000000000000000000000000000000..190a222338b24b7edac76c72d07df0b2cbd7d9be --- /dev/null +++ b/go/glide.lock @@ -0,0 +1,61 @@ +hash: b8f18ce6784bd3fadd9fed0b8443e7b658234ea785ae1f220723ae2c1f652aa7 +updated: 2017-06-27T14:05:48.925262819+08:00 +imports: +- name: github.com/coreos/etcd + version: 61fc123e7a8b14a0a258aa3f5c4159861b1ec2e7 + subpackages: + - auth/authpb + - clientv3 + - clientv3/concurrency + - etcdserver/api/v3rpc/rpctypes + - etcdserver/etcdserverpb + - mvcc/mvccpb +- name: github.com/golang/protobuf + version: 4bd1920723d7b7c925de087aa32e2187708897f7 + subpackages: + - jsonpb + - proto +- name: github.com/golang/snappy + version: 553a641470496b2327abcac10b36396bd98e45c9 +- name: github.com/namsral/flag + version: 71ceffbeb0ba60fccc853971bb3ed4d7d90bfd04 +- name: github.com/PaddlePaddle/recordio + version: edfb82af0739c84f241c87390ec5649c7b28c129 +- name: github.com/sirupsen/logrus + version: 202f25545ea4cf9b191ff7f846df5d87c9382c2b +- name: golang.org/x/net + version: c8c74377599bd978aee1cf3b9b63a8634051cec2 + subpackages: + - context + - http2 + - http2/hpack + - idna + - internal/timeseries + - lex/httplex + - trace +- name: golang.org/x/sys + version: f7928cfef4d09d1b080aa2b6fd3ca9ba1567c733 + subpackages: + - unix +- name: golang.org/x/text + version: 4e9ab9ee170f2a39bd66c92b3e0a47ff47a4bc77 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: google.golang.org/grpc + version: 8050b9cbc271307e5a716a9d782803d09b0d6f2d + subpackages: + - codes + - credentials + - grpclog + - internal + - keepalive + - metadata + - naming + - peer + - stats + - tap + - transport +testImports: [] diff --git a/go/glide.yaml b/go/glide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05c5d15ca22b6a3d85bee8e1f31d222034ce5314 --- /dev/null +++ b/go/glide.yaml @@ -0,0 +1,12 @@ +package: github.com/PaddlePaddle/Paddle/go +import: +- package: github.com/PaddlePaddle/recordio +- package: github.com/coreos/etcd + version: ^3.2.1 + subpackages: + - clientv3 + - clientv3/concurrency +- package: github.com/namsral/flag + version: ^1.7.4-pre +- package: github.com/sirupsen/logrus + version: ^1.0.0 diff --git a/go/master/c/CMakeLists.txt b/go/master/c/CMakeLists.txt index a4e92635bab8f0dbc5d91c7131fb0f8d1f5a6363..d900850be04e3f385cc7fbf341ef0bb9fe53e789 100644 --- a/go/master/c/CMakeLists.txt +++ b/go/master/c/CMakeLists.txt @@ -1,3 +1 @@ -cmake_minimum_required(VERSION 3.0) - -go_library(paddle_master SHARED) +go_library(paddle_master SHARED DEPS paddle_go_optimizer) diff --git a/go/master/c/client.go b/go/master/c/client.go index 9e35e986002c0ae3b7593150ece96dba29a1521b..31f431197454c2ec6a25624d37b60876d99f0087 100644 --- a/go/master/c/client.go +++ b/go/master/c/client.go @@ -104,11 +104,22 @@ func paddle_set_dataset(client C.paddle_master_client, path **C.char, size C.int return C.PADDLE_MASTER_OK } +// return value: +// 0:ok +// -1:error //export paddle_next_record func paddle_next_record(client C.paddle_master_client, record **C.uchar) C.int { c := get(client) - r := c.NextRecord() + r, err := c.NextRecord() + if err != nil { + // Error + // TODO: return the type of error? + *record = (*C.uchar)(nullPtr) + return -1 + } + if len(r) == 0 { + // Empty record *record = (*C.uchar)(nullPtr) return 0 } diff --git a/go/master/client.go b/go/master/client.go index d3bea49d0a8166420e83478076cc7bc81e48598d..05383f1bf40c0e2b1f974e802ee8fd6aac109b00 100644 --- a/go/master/client.go +++ b/go/master/client.go @@ -11,7 +11,12 @@ import ( // Client is the client of the master server. type Client struct { conn *connection.Conn - ch chan []byte + ch chan record +} + +type record struct { + r []byte + err error } // NewClient creates a new Client. @@ -21,7 +26,7 @@ type Client struct { func NewClient(addrCh <-chan string, bufSize int) *Client { c := &Client{} c.conn = connection.New() - c.ch = make(chan []byte, bufSize) + c.ch = make(chan record, bufSize) go c.monitorMaster(addrCh) go c.getRecords() return c @@ -46,10 +51,11 @@ func (c *Client) getRecords() { s := recordio.NewRangeScanner(f, &chunk.Index, -1, -1) for s.Scan() { - c.ch <- s.Record() + c.ch <- record{s.Record(), nil} } if s.Err() != nil { + c.ch <- record{nil, s.Err()} log.Errorln(err, chunk.Path) } @@ -116,6 +122,7 @@ func (c *Client) taskFinished(taskID int) error { // // NextRecord will block until the next record is available. It is // thread-safe. -func (c *Client) NextRecord() []byte { - return <-c.ch +func (c *Client) NextRecord() ([]byte, error) { + r := <-c.ch + return r.r, r.err } diff --git a/go/master/client_test.go b/go/master/client_test.go index c00aeebfd5d1fef6de4a8c67bf7f998a42ee863b..6666d3860c412daa8711fbfa2d729a261b3fd887 100644 --- a/go/master/client_test.go +++ b/go/master/client_test.go @@ -68,12 +68,17 @@ func TestNextRecord(t *testing.T) { for pass := 0; pass < 50; pass++ { received := make(map[byte]bool) for i := 0; i < total; i++ { - r := c.NextRecord() + r, err := c.NextRecord() + if err != nil { + t.Fatal(pass, i, "Read error:", err) + } + if len(r) != 1 { - t.Fatal("Length should be 1.", r) + t.Fatal(pass, i, "Length should be 1.", r) } + if received[r[0]] { - t.Fatal("Received duplicate.", received, r) + t.Fatal(pass, i, "Received duplicate.", received, r) } received[r[0]] = true } diff --git a/go/master/etcd_client.go b/go/master/etcd_client.go index e27c014792f31ca27fe1a1636d69acccc4206ea3..04c1394e963d1eb541b80b91407fb55b0d1e1f2a 100644 --- a/go/master/etcd_client.go +++ b/go/master/etcd_client.go @@ -50,7 +50,7 @@ func NewEtcdClient(endpoints []string, addr string, lockPath, addrPath, statePat lock := concurrency.NewMutex(sess, lockPath) // It's fine for the lock to get stuck, in this case we have // multiple master servers running (only configured to have - // one master running, but split-brain problem may cuase + // one master running, but split-brain problem may cause // multiple master servers running), and the cluster management // software will kill one of them. log.Debugf("Trying to acquire lock at %s.", lockPath) @@ -98,7 +98,7 @@ func (e *EtcdClient) Save(state []byte) error { // We lost the master lock and can not acquire // it back, it means some other master is // already started. We don't want cluster - // managment system to kill the master server + // management system to kill the master server // who is holding the lock and running // correctly. So the most feasible solution is // to kill current master server. The current diff --git a/go/pserver/cclient/CMakeLists.txt b/go/pserver/cclient/CMakeLists.txt deleted file mode 100644 index 7fe74c62f109b186eb43383b78f30478b9be74c1..0000000000000000000000000000000000000000 --- a/go/pserver/cclient/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf) -go_library(paddle_pserver_cclient STATIC) -if(WITH_TESTING) - add_subdirectory(test) -endif() diff --git a/go/pserver/cclient/test/testdata/optimizer.pb.txt b/go/pserver/cclient/test/testdata/optimizer.pb.txt deleted file mode 100644 index 27c8a584df40ab714edfd730f0ff7b7bd3783964..0000000000000000000000000000000000000000 Binary files a/go/pserver/cclient/test/testdata/optimizer.pb.txt and /dev/null differ diff --git a/go/pserver/client/c/CMakeLists.txt b/go/pserver/client/c/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..93a0a27f858f8654e0a6114abe7e326b086b8bf9 --- /dev/null +++ b/go/pserver/client/c/CMakeLists.txt @@ -0,0 +1,8 @@ +cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf) +target_link_libraries(paddle_go_optimizer stdc++ m) +go_library(paddle_pserver_cclient STATIC DEPS paddle_go_optimizer) +if(WITH_TESTING) + # FIXME: this test requires pserver which is not managed by the test + # we need some kind of e2e testing machanism. + # add_subdirectory(test) +endif() diff --git a/go/pserver/cclient/cclient.go b/go/pserver/client/c/cclient.go similarity index 88% rename from go/pserver/cclient/cclient.go rename to go/pserver/client/c/cclient.go index bbaf43d9f1434a278568bc110a709718b9b8c222..7ddaceb7ed33db32e19a191402100a0c0efa241a 100644 --- a/go/pserver/cclient/cclient.go +++ b/go/pserver/client/c/cclient.go @@ -30,15 +30,16 @@ import ( "unsafe" "github.com/PaddlePaddle/Paddle/go/pserver" + "github.com/PaddlePaddle/Paddle/go/pserver/client" log "github.com/sirupsen/logrus" ) var nullPtr = unsafe.Pointer(uintptr(0)) var mu sync.Mutex -var handleMap = make(map[C.paddle_pserver_client]*pserver.Client) +var handleMap = make(map[C.paddle_pserver_client]*client.Client) var curHandle C.paddle_pserver_client -func add(c *pserver.Client) C.paddle_pserver_client { +func add(c *client.Client) C.paddle_pserver_client { mu.Lock() defer mu.Unlock() client := curHandle @@ -47,13 +48,13 @@ func add(c *pserver.Client) C.paddle_pserver_client { return client } -func get(client C.paddle_pserver_client) *pserver.Client { +func get(client C.paddle_pserver_client) *client.Client { mu.Lock() defer mu.Unlock() return handleMap[client] } -func remove(client C.paddle_pserver_client) *pserver.Client { +func remove(client C.paddle_pserver_client) *client.Client { mu.Lock() defer mu.Unlock() h := handleMap[client] @@ -80,9 +81,9 @@ func (s selector) Select() bool { return bool(s) } -type lister []pserver.Server +type lister []client.Server -func (l lister) List() []pserver.Server { +func (l lister) List() []client.Server { return l } @@ -90,19 +91,22 @@ func (l lister) List() []pserver.Server { func paddle_new_pserver_client(addrs *C.char, selected int) C.paddle_pserver_client { a := C.GoString(addrs) as := strings.Split(a, ",") - servers := make([]pserver.Server, len(as)) + servers := make([]client.Server, len(as)) for i := range as { servers[i].Index = i servers[i].Addr = as[i] } - c := pserver.NewClient(lister(servers), len(as), selector(selected != 0)) + c := client.NewClient(lister(servers), len(as), selector(selected != 0)) return add(c) } //export paddle_new_etcd_pserver_client -func paddle_new_etcd_pserver_client(etcd_addr *C.char) C.paddle_pserver_client { - // TODO(helin): fault tolerant pserver client using etcd. - panic("not implemented.") +func paddle_new_etcd_pserver_client(etcd_endpoints *C.char, selected int) C.paddle_pserver_client { + // TODO(Longfei: use etcd lock to decide which trainer to initialize the parameters) + addr := C.GoString(etcd_endpoints) + etcd_client := client.NewEtcd(addr) + c := client.NewClient(etcd_client, etcd_client.Desired(), selector(selected != 0)) + return add(c) } //export paddle_pserver_client_release diff --git a/go/pserver/cclient/test/CMakeLists.txt b/go/pserver/client/c/test/CMakeLists.txt similarity index 81% rename from go/pserver/cclient/test/CMakeLists.txt rename to go/pserver/client/c/test/CMakeLists.txt index f287f850719afecf918f6a53f6528d1d15ff4672..dce8645ce753f6a14b298726c714be18de3834e4 100644 --- a/go/pserver/cclient/test/CMakeLists.txt +++ b/go/pserver/client/c/test/CMakeLists.txt @@ -1,2 +1,2 @@ -cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient) +cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient paddle_go_optimizer) add_style_check_target(test_cclient test_cclient.c) diff --git a/go/pserver/cclient/test/test_cclient.c b/go/pserver/client/c/test/test_cclient.c similarity index 98% rename from go/pserver/cclient/test/test_cclient.c rename to go/pserver/client/c/test/test_cclient.c index b16769b433e72188d411ba1bf2586a3702434324..8eababbe33914d25f1eb91b991e11eaacd2e4716 100644 --- a/go/pserver/cclient/test/test_cclient.c +++ b/go/pserver/client/c/test/test_cclient.c @@ -51,7 +51,7 @@ int main() { char *config_proto; size_t config_proto_len = 0; ssize_t nread; - FILE *fp = fopen("testdata/optimizer.pb.txt", "r"); + FILE *fp = fopen("testdata/optimizer.pb", "r"); if (!fp) { fail(); } diff --git a/go/pserver/cclient/test/test_mnist.py b/go/pserver/client/c/test/test_mnist.py similarity index 100% rename from go/pserver/cclient/test/test_mnist.py rename to go/pserver/client/c/test/test_mnist.py diff --git a/go/pserver/cclient/test/test_train.py b/go/pserver/client/c/test/test_train.py similarity index 100% rename from go/pserver/cclient/test/test_train.py rename to go/pserver/client/c/test/test_train.py diff --git a/go/pserver/client/c/test/testdata/optimizer.pb b/go/pserver/client/c/test/testdata/optimizer.pb new file mode 100644 index 0000000000000000000000000000000000000000..27dd3bc5f19e2964b4b674cff8860233cbdb445a Binary files /dev/null and b/go/pserver/client/c/test/testdata/optimizer.pb differ diff --git a/go/pserver/client.go b/go/pserver/client/client.go similarity index 92% rename from go/pserver/client.go rename to go/pserver/client/client.go index 6938b9d5ce6f6d73c05bd6e3154777023965c319..aa8bfe30c26fcc0875ad479ecd562700ccefa5a3 100644 --- a/go/pserver/client.go +++ b/go/pserver/client/client.go @@ -1,4 +1,4 @@ -package pserver +package client import ( "errors" @@ -7,6 +7,7 @@ import ( "time" "github.com/PaddlePaddle/Paddle/go/connection" + "github.com/PaddlePaddle/Paddle/go/pserver" log "github.com/sirupsen/logrus" ) @@ -105,7 +106,7 @@ func (c *Client) BeginInitParams() bool { } // InitParam initializes the parameter on parameter servers. -func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error { +func (c *Client) InitParam(paramWithConfigs pserver.ParameterWithConfig) error { return c.pservers[c.partition(paramWithConfigs.Param.Name)].Call("Service.InitParam", paramWithConfigs, nil) } @@ -123,13 +124,13 @@ func (c *Client) FinishInitParams() error { // SendGrads sends gradients to parameter servers for updating // parameters. -func (c *Client) SendGrads(grads []Gradient) error { +func (c *Client) SendGrads(grads []pserver.Gradient) error { if len(grads) == 0 { return errors.New("no gradient received") } errCh := make(chan error, len(grads)) for _, g := range grads { - go func(g Gradient) { + go func(g pserver.Gradient) { err := c.pservers[c.partition(g.Name)].Call("Service.SendGrad", g, nil) errCh <- err }(g) @@ -151,7 +152,7 @@ func (c *Client) SendGrads(grads []Gradient) error { type result struct { idx int - param Parameter + param pserver.Parameter err error } @@ -170,12 +171,12 @@ func (r results) Swap(i int, j int) { } // GetParams gets parameters from parameter servers. -func (c *Client) GetParams(names []string) ([]Parameter, error) { +func (c *Client) GetParams(names []string) ([]pserver.Parameter, error) { rCh := make(chan result, len(names)) for idx, name := range names { go func(name string, idx int) { - var parameter Parameter + var parameter pserver.Parameter err := c.pservers[c.partition(name)].Call("Service.GetParam", name, ¶meter) rCh <- result{idx: idx, param: parameter, err: err} }(name, idx) @@ -196,7 +197,7 @@ func (c *Client) GetParams(names []string) ([]Parameter, error) { } sort.Sort(rs) - ps := make([]Parameter, len(rs)) + ps := make([]pserver.Parameter, len(rs)) for i := range rs { ps[i] = rs[i].param } diff --git a/go/pserver/client_test.go b/go/pserver/client/client_test.go similarity index 54% rename from go/pserver/client_test.go rename to go/pserver/client/client_test.go index a248a3fb696a1e03b799f89afceb255de68662b1..29b400812c9dc3a5f44700eacbf7ba043248f2f2 100644 --- a/go/pserver/client_test.go +++ b/go/pserver/client/client_test.go @@ -1,6 +1,7 @@ -package pserver_test +package client_test import ( + "context" "io/ioutil" "net" "net/http" @@ -8,15 +9,25 @@ import ( "strconv" "strings" "testing" + "time" "github.com/PaddlePaddle/Paddle/go/pserver" + "github.com/PaddlePaddle/Paddle/go/pserver/client" + "github.com/coreos/etcd/clientv3" + log "github.com/sirupsen/logrus" ) -const numPserver = 10 +const ( + numPserver = 10 + etcdEndpoints = "127.0.0.1:2379" + timeout = 2 * time.Second +) -var port [numPserver]int +var pserverClientPorts [numPserver]int -func init() { +// this function init pserver client and return their ports in an array. +func initClient() [numPserver]int { + var ports [numPserver]int for i := 0; i < numPserver; i++ { l, err := net.Listen("tcp", ":0") if err != nil { @@ -28,7 +39,7 @@ func init() { if err != nil { panic(err) } - port[i] = p + ports[i] = p go func(l net.Listener) { s, err := pserver.NewService(0) @@ -49,6 +60,31 @@ func init() { } }(l) } + return ports +} + +func initNativeClient() { + pserverClientPorts = initClient() +} + +func initEtcdClient() { + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{etcdEndpoints}, + DialTimeout: time.Second * time.Duration(1), + }) + if err != nil { + log.Errorf("err %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + client.Delete(ctx, pserver.PsDesired) + client.Delete(ctx, pserver.PsPath) + client.Put(ctx, pserver.PsDesired, strconv.Itoa(numPserver)) + ports := initClient() + for i := 0; i < numPserver; i++ { + client.Put(ctx, pserver.PsPath+strconv.Itoa(i), ":"+strconv.Itoa(ports[i])) + } + cancel() + client.Close() } type selector bool @@ -57,25 +93,20 @@ func (s selector) Select() bool { return bool(s) } -type lister []pserver.Server +type lister []client.Server -func (l lister) List() []pserver.Server { +func (l lister) List() []client.Server { return l } -func TestClientFull(t *testing.T) { - servers := make([]pserver.Server, numPserver) - for i := 0; i < numPserver; i++ { - servers[i] = pserver.Server{Index: i, Addr: ":" + strconv.Itoa(port[i])} - } - c := pserver.NewClient(lister(servers), len(servers), selector(true)) +func ClientTest(t *testing.T, c *client.Client) { selected := c.BeginInitParams() if !selected { t.Fatal("should be selected.") } const numParameter = 100 - config, err := ioutil.ReadFile("./cclient/test/testdata/optimizer.pb.txt") + config, err := ioutil.ReadFile("./c/test/testdata/optimizer.pb") if err != nil { t.Fatalf("read optimizer proto failed") } @@ -129,3 +160,21 @@ func TestClientFull(t *testing.T) { } } } + +func TestNativeClient(t *testing.T) { + initNativeClient() + servers := make([]client.Server, numPserver) + for i := 0; i < numPserver; i++ { + servers[i] = client.Server{Index: i, Addr: ":" + strconv.Itoa(pserverClientPorts[i])} + } + c1 := client.NewClient(lister(servers), len(servers), selector(true)) + ClientTest(t, c1) +} + +// TODO: tmperary disable etcdClient test for dependency of etcd) +func EtcdClient(t *testing.T) { + initEtcdClient() + etcd_client := client.NewEtcd(etcdEndpoints) + c2 := client.NewClient(etcd_client, etcd_client.Desired(), selector(true)) + ClientTest(t, c2) +} diff --git a/go/pserver/client/etcd_client.go b/go/pserver/client/etcd_client.go new file mode 100644 index 0000000000000000000000000000000000000000..1fd3479aa88ccbbe7c5067da1e9886b65352e847 --- /dev/null +++ b/go/pserver/client/etcd_client.go @@ -0,0 +1,125 @@ +package client + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/PaddlePaddle/Paddle/go/pserver" + "github.com/coreos/etcd/clientv3" + log "github.com/sirupsen/logrus" +) + +const ( + DefaultEtcdTimeout time.Duration = 5 * time.Second +) + +// EtcdClient is used by pserver client that is a part of trainer process. +// TODO: +// 1. add watcher to watch the change state of pservers) +// 1. add etcd lock) +type EtcdClient struct { + client *clientv3.Client + timeout time.Duration + endpoints []string +} + +// Desired read ps desired number from etcd. +func (p *EtcdClient) Desired() int { + var psDesired int + for { + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + resp, err := p.client.Get(ctx, pserver.PsDesired) + cancel() + if err != nil { + log.Errorf("Get ps dresire number failed! recnnectiong..., %v", err) + time.Sleep(p.timeout) + continue + } + + kvs := resp.Kvs + if len(kvs) == 0 { + log.Infoln("Waiting for ps desired registered ...") + time.Sleep(p.timeout) + continue + } + + psDesired, err = strconv.Atoi(string(resp.Kvs[0].Value)) + if err != nil { + log.Errorf("psDesired %s invalid %v", psDesired, err) + time.Sleep(p.timeout) + continue + } + + log.Debugf("Get psDesired number: %d", psDesired) + break + } + return psDesired +} + +// List return the pserver list read from etcd. +func (p *EtcdClient) List() []Server { + psDesired := p.Desired() + + servers := make([]Server, psDesired) + for { + for i := 0; i < psDesired; i++ { + ctx, cancel := context.WithTimeout(context.Background(), p.timeout) + cancel() + psKey := pserver.PsPath + strconv.Itoa(i) + log.Debugf("checking %s", psKey) + resp, err := p.client.Get(ctx, psKey) + if err != nil { + log.Infof("Get psKey= %s error, %v", psKey, err) + time.Sleep(p.timeout) + continue + } + kvs := resp.Kvs + if len(kvs) == 0 { + log.Infof("Waiting for ps addr registered ...") + time.Sleep(p.timeout) + continue + } + + psAddr := string(resp.Kvs[0].Value) + // TODO(Longfei) check the ps address + if psAddr == "" { + log.Infof("Get psKey = %s, psAddr is empty", psKey) + time.Sleep(p.timeout) + continue + } + log.Infof("got value (%s) for key: %s", psAddr, psKey) + servers[i].Index = i + servers[i].Addr = psAddr + } + break + } + return servers +} + +// NewEtcd create a etcd client to return the state of pserver on etcd. +func NewEtcd(endpoints string) *EtcdClient { + ep := strings.Split(endpoints, ",") + var cli *clientv3.Client + var err error + for { + cli, err = clientv3.New(clientv3.Config{ + Endpoints: ep, + DialTimeout: DefaultEtcdTimeout, + }) + if err != nil { + log.Errorf("Init etcd connection failed: %v", err) + time.Sleep(DefaultEtcdTimeout) + continue + } + break + } + log.Infof("Connected to etcd: %s\n", endpoints) + client := &EtcdClient{ + client: cli, + timeout: DefaultEtcdTimeout, + endpoints: ep, + } + return client +} diff --git a/go/pserver/etcd_client.go b/go/pserver/etcd_client.go index 4d88243edd4aa817ddc263ba316a3f6be9e1e67f..37b8d522c1bd07acb41b9515a6d9bc15eae9aa32 100644 --- a/go/pserver/etcd_client.go +++ b/go/pserver/etcd_client.go @@ -13,6 +13,13 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + // PsDesired is etcd path for store desired pserver count + PsDesired = "/ps_desired" + // PsAddr is the base dir for pserver to store their addr + PsPath = "/ps/" +) + // EtcdClient is the etcd client that the pserver uses for fault // tolerance, service registry and coordination. type EtcdClient struct { @@ -68,7 +75,7 @@ func (e *EtcdClient) Register() (int, error) { // it at the same time. for { ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _, err := e.initDesiredPsercers(ctx, e.numPservers) + _, err := e.initDesiredPservers(ctx, e.numPservers) cancel() if err != nil { log.Warn(err) @@ -120,7 +127,7 @@ func (e *EtcdClient) Register() (int, error) { return pserverIdx, nil } -func (e *EtcdClient) initDesiredPsercers(ctx context.Context, numPservers int) (*clientv3.TxnResponse, error) { +func (e *EtcdClient) initDesiredPservers(ctx context.Context, numPservers int) (*clientv3.TxnResponse, error) { return concurrency.NewSTM(e.etcdClient, func(c concurrency.STM) error { dsStr := c.Get(PsDesired) if dsStr == "" { @@ -136,7 +143,7 @@ func (e *EtcdClient) registerPserverEtcd(ctx context.Context) (int, error) { _, err := concurrency.NewSTM(e.etcdClient, func(c concurrency.STM) error { registered := false for i := 0; i < e.desired; i++ { - psKey := "/ps/" + strconv.Itoa(i) + psKey := PsPath + strconv.Itoa(i) log.Debugf("checking %s", psKey) ps := c.Get(psKey) log.Debugf("got value (%s) for key: %s", ps, psKey) diff --git a/go/pserver/optimizer.go b/go/pserver/optimizer.go index b4a040f46bff5c25b193d41e5d36b59762891574..54d108209402c27e79a9948f60ecbdadeffc7d9b 100644 --- a/go/pserver/optimizer.go +++ b/go/pserver/optimizer.go @@ -2,7 +2,7 @@ package pserver // #cgo CFLAGS: -I ../../ // //FIXME: ldflags contain "build" path -// #cgo LDFLAGS: ../../build/go/pserver/cclient/libpaddle_go_optimizer.a -lstdc++ +// #cgo LDFLAGS: ${SRCDIR}/../../build/go/pserver/client/c/libpaddle_go_optimizer.a -lstdc++ -lm // #include "paddle/optimizer/optimizer.h" // #include // #include @@ -56,8 +56,8 @@ func newOptimizer(paramWithConfigs ParameterWithConfig) *optimizer { func (o *optimizer) GetWeights() []byte { var buffer unsafe.Pointer - buffer_len := C.paddle_optimizer_get_weights(o.opt, &buffer) - return cArrayToSlice(buffer, int(buffer_len)*C.sizeof_float) + bufferLen := C.paddle_optimizer_get_weights(o.opt, &buffer) + return cArrayToSlice(buffer, int(bufferLen)*C.sizeof_float) } func (o *optimizer) UpdateParameter(g Gradient) error { diff --git a/go/pserver/optimizer_test.go b/go/pserver/optimizer_test.go index 368047d6f89e080016909efbc5bd090c42530bfd..0b2f4cfa41a630645c128ac13826de9d8b1d521b 100644 --- a/go/pserver/optimizer_test.go +++ b/go/pserver/optimizer_test.go @@ -11,7 +11,7 @@ func TestOptimizerCreateRelease(t *testing.T) { ElementType: Int32, } p.Content = []byte{1, 3} - config, err := ioutil.ReadFile("./cclient/test/testdata/optimizer.pb.txt") + config, err := ioutil.ReadFile("./client/c/test/testdata/optimizer.pb") if err != nil { t.Fatalf("read optimizer proto failed") } diff --git a/go/pserver/service.go b/go/pserver/service.go index e15a4e5a58a3bb1a154157b1212d141478e96231..ad16a5708d10bdcb5189a1e1e8abf13c54a72265 100644 --- a/go/pserver/service.go +++ b/go/pserver/service.go @@ -10,8 +10,10 @@ import ( type ElementType int const ( + // AlreadyInitialized is true if pserver is initialized AlreadyInitialized = "pserver already initialized" - Uninitialized = "pserver not fully initialized" + // Uninitialized is true if pserver not fully initialized + Uninitialized = "pserver not fully initialized" ) // Supported element types @@ -24,9 +26,6 @@ const ( Float64 ) -// PsDesired is etcd path for store desired pserver count -const PsDesired = "/ps_desired" - // Parameter is a piece of data to sync with the parameter server. type Parameter struct { Name string @@ -58,7 +57,7 @@ func NewService(idx int) (*Service, error) { s := &Service{ idx: idx, } - s.optMap = make(map[string]*optimizer) + s.optMap = make(map[string]*optimizer) s.initialized = make(chan struct{}) return s, nil } diff --git a/go/pserver/service_test.go b/go/pserver/service_test.go index f86619447c28b5be8071d28a127b48768939261b..b6d20d2c8b7ba0ccd7ab46669a597a21dc11c381 100644 --- a/go/pserver/service_test.go +++ b/go/pserver/service_test.go @@ -10,6 +10,10 @@ import ( "github.com/PaddlePaddle/Paddle/go/pserver" ) +const ( + OptimizerConfig = "./client/c/test/testdata/optimizer.pb" +) + func TestServiceFull(t *testing.T) { s, err := pserver.NewService(0) if err != nil { @@ -19,7 +23,7 @@ func TestServiceFull(t *testing.T) { p.Name = "param_a" p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0} p.ElementType = pserver.Int32 - config, err := ioutil.ReadFile("./cclient/test/testdata/optimizer.pb.txt") + config, err := ioutil.ReadFile(OptimizerConfig) if err != nil { t.Fatalf("read optimizer proto failed") } @@ -149,7 +153,7 @@ func TestBlockUntilInitialized(t *testing.T) { p.Name = "param_a" p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0} p.ElementType = pserver.Int32 - config, err := ioutil.ReadFile("./cclient/test/testdata/optimizer.pb.txt") + config, err := ioutil.ReadFile(OptimizerConfig) if err != nil { t.Fatalf("read optimizer proto failed") } diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index 39d8aa075bc072d37dc8df67746f0d2b503418a6..84da89a1422b6095b995744cebb6a3af98a071c6 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -66,6 +66,7 @@ SWIG_LINK_LIBRARIES(swig_paddle paddle_trainer_lib paddle_network paddle_parameter + paddle_optimizer paddle_math paddle_utils paddle_proto diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 6aa6b9bc2db6a223dd8562b76ba9d777206bfd40..4409c6feae218222b7c0216760cebe4ae8e235cb 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -2,6 +2,17 @@ cc_library(ddim SRCS ddim.cc) cc_test(ddim_test SRCS ddim_test.cc DEPS ddim) nv_test(dim_test SRCS dim_test.cu DEPS ddim) +cc_test(tensor_test SRCS tensor_test.cc DEPS ddim) cc_test(variable_test SRCS variable_test.cc) cc_test(scope_test SRCS scope_test.cc) cc_test(enforce_test SRCS enforce_test.cc) +proto_library(attr_type SRCS attr_type.proto) +proto_library(op_proto SRCS op_proto.proto DEPS attr_type) +cc_test(op_proto_test SRCS op_proto_test.cc DEPS op_proto protobuf) +proto_library(op_desc SRCS op_desc.proto DEPS attr_type) +cc_test(op_desc_test SRCS op_desc_test.cc DEPS op_desc protobuf) +cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_proto op_desc) +py_proto_compile(framework_py_proto SRCS attr_type.proto op_proto.proto op_desc.proto) +# Generate an empty __init__.py to make framework_py_proto as a valid python module. +add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) +add_dependencies(framework_py_proto framework_py_proto_init) diff --git a/paddle/framework/attr_checker.h b/paddle/framework/attr_checker.h new file mode 100644 index 0000000000000000000000000000000000000000..c0c33d81149ac2fc2a9a57d90931ef32375fe1d0 --- /dev/null +++ b/paddle/framework/attr_checker.h @@ -0,0 +1,119 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "paddle/framework/enforce.h" + +namespace paddle { +namespace framework { + +typedef boost::variant, + std::vector, std::vector> + Attribute; +typedef std::unordered_map AttributeMap; + +// check whether a value(attribute) fit a certain limit +template +class LargerThanChecker { + public: + LargerThanChecker(T lower_bound) : lower_bound_(lower_bound) {} + void operator()(T& value) const { + PADDLE_ENFORCE(value > lower_bound_, "larger_than check fail"); + } + + private: + T lower_bound_; +}; + +// we can provide users more common Checker, like 'LessThanChecker', +// 'BetweenChecker'... + +template +class DefaultValueSetter { + public: + DefaultValueSetter(T default_value) : default_value_(default_value) {} + void operator()(T& value) const { value = default_value_; } + + private: + T default_value_; +}; + +// check whether a certain attribute fit its limits +// an attribute can have more than one limits +template +class TypedAttrChecker { + typedef std::function ValueChecker; + + public: + TypedAttrChecker(const std::string& attr_name) : attr_name_(attr_name) {} + + TypedAttrChecker& LargerThan(const T& lower_bound) { + value_checkers_.push_back(LargerThanChecker(lower_bound)); + return *this; + } + + // we can add more common limits, like LessThan(), Between()... + + TypedAttrChecker& SetDefault(const T& default_value) { + PADDLE_ENFORCE(default_value_setter_.empty(), + "%s can't have more than one default value!", attr_name_); + default_value_setter_.push_back(DefaultValueSetter(default_value)); + return *this; + } + + // allow users provide their own checker + TypedAttrChecker& AddCustomChecker(const ValueChecker& checker) { + value_checkers_.push_back(checker); + return *this; + } + + void operator()(AttributeMap& attr_map) const { + if (!attr_map.count(attr_name_)) { + // user do not set this attr + PADDLE_ENFORCE(!default_value_setter_.empty(), + "Attribute '%s' is required!", attr_name_); + // default_value_setter_ has no more than one element + T val; + (default_value_setter_[0])(val); + attr_map[attr_name_] = val; + } + Attribute& attr = attr_map.at(attr_name_); + T& attr_value = boost::get(attr); + for (const auto& checker : value_checkers_) { + checker(attr_value); + } + } + + private: + std::string attr_name_; + std::vector value_checkers_; + std::vector default_value_setter_; +}; + +// check whether op's all attributes fit their own limits +class OpAttrChecker { + typedef std::function AttrChecker; + + public: + template + TypedAttrChecker& AddAttrChecker(const std::string& attr_name) { + attr_checkers_.push_back(TypedAttrChecker(attr_name)); + AttrChecker& checker = attr_checkers_.back(); + return *(checker.target>()); + } + + void Check(AttributeMap& attr_map) const { + for (const auto& checker : attr_checkers_) { + checker(attr_map); + } + } + + private: + std::vector attr_checkers_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/attr_type.proto b/paddle/framework/attr_type.proto new file mode 100644 index 0000000000000000000000000000000000000000..2d8e0476d710b7ba987d085d828ca13a4ee23707 --- /dev/null +++ b/paddle/framework/attr_type.proto @@ -0,0 +1,28 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +syntax="proto2"; +package paddle.framework; + +// Attribute Type for paddle's Op. +// Op contains many attributes. Each type of attributes could be different. +// The AttrType will be shared between AttrDesc and AttrProto. +enum AttrType { + INT = 0; + FLOAT = 1; + STRING = 2; + INTS = 3; + FLOATS = 4; + STRINGS = 5; +} \ No newline at end of file diff --git a/paddle/framework/net_design.md b/paddle/framework/net_design.md new file mode 100644 index 0000000000000000000000000000000000000000..a5f0483081e8a03b2d001a551fcc02bbd392016d --- /dev/null +++ b/paddle/framework/net_design.md @@ -0,0 +1,250 @@ +# Network Design + +`Network` is the container and controller of a set of operators, +user can build a real network from a `NetDesc` which is a protobuf message +and use `Network.Run()` to run all the operators in the network. + +A network object knows all Operators belonging to this network. Variables, +which are inputs and outputs of these operators, +are created and managed by a hierarchy of Scope objects. + +# API + +## Net +To make the `Network` extendable, a base class is defined like this + +```c++ +// operator's index stored in a network. +typedef int OpIndex; + +// The minimum a network should be implemented. +class Net { + public: + // run all the operators and return success(true) or not, with all the + // variables are located in `scope`. `context` describes the detail execution + // environment for ops. `begin` and `end` specify the scope of `ops_` to run, + // If no positive indexes are provided, all operators in `ops_` will run. + virtual Error Run(Scope *scope, OpContext *context, OpIndex begin = -1, + OpIndex end = -1) const = 0; + + // Add an Operator according to `def`. + virtual OpIndex AddOp(const proto::OpDef &def) = 0; + + // Add optimizer operators acctording to `attrs`. + virtual Error AddOptimizerOps(const OptAttrs &attrs) = 0; + + // Add backward operators. + virtual Error AddBackwardOps() = 0; + + // Infer the shapes of variables required by operators in the network. The + // `scope` will be mutated according to the inferred shapes. + + static std::unique_ptr Create(const NetDesc &def = NetDesc()); +}; +``` + +All network implementations should build networks from a protobuf message which +describes the structure of a real network; `Run` method should be implemented by +all implementations to offer a universal method to forward or backward compute a network. + +`Net::Create` is a method of factory pattern and can be implemented like + +```c++ +std::unique Net::Create(const NetDesc& def) { + switch (def.model_type()) { + case NN: + return new Network(def); + case Recursive: + return new RecursiveNet(def); + case Recurrent: + return new RecurrentNet(def); + } + return nullptr; +} +``` + +Network is designed as the container of operators. to make it more extendable, +we decouple it from the related variable resources. + +`Run(Scope* scope)` takes the scope as a argument so that it can run in different scopes. + +Finally, `Net` can be used as followed + +```c++ +Scope default_scope; +OpContext default_context; +auto net = Net::CreateNet(def); + +if (net) { + net.Run(&default_scope, &default_context); +} +``` + +## `PlainNet` as a simple implementation of `BaseNet` + +A very basic implementation is as follows. All it does is simply to run every operators in sequence. + +```c++ +class PlainNet : public Net { + public: + // Create a network describe by `def`. NetDesc is the definition of a network. + PlainNet(const NetDesc &def); + + // Infer all the operators' input and output varialbes' shapes, will be called before every mini-batch + training. + virtual Error InferShape(Scope *scope) override; + + // Run all the operators with the `scope`, if no scope is provided, default + // scope will be used instead. If no OpContext is provicded, default context will be used. + virtual Error Run(Scope *scope = nullptr, OpContext *context=nullptr, OpIndex begin = -1, + OpIndex end = -1) const override; + + virtual OpIndex AddOp(const proto::OpDef &def) override; + + virtual Error AddOptimizerOps(const OptAttrs &attrs) override; + + virtual Error AddBackwardOps() override; + + protected: + // Create operators accordding to `def`, will be called by the constructor. + Error BuildNet(const NetDesc &def); + + // Add a operator which is identified as `type` and has attributes described + // in `attrs`, the `inputs` are the keys of readonly input variables, + // `outputs` are keys of mutable output variables. An `OpIndex` will be + // returned to indicate the offset of the new operator in `ops_`. + OpIndex AddOp(const std::string &type, const std::vector &inputs, + const std::vector &outputs, + const OprAttr &attrs = OprAttr()); + + private: + // the operators owned by `Network`. + std::vector ops_; +}; +``` + +`PlainNet` will create operators so that a private member `ops_` is defined, +the operators are created by `CreateNet`, and each operator is created by `AddOp`. + + +## PlainNet Usage +`PlainNet` can be used to define and run a network as follows + +```c++ +// create an empty scope located on CPU device. +Scope scope(CPUPlace()); + +// create and init variables described in `net_desc`. +scope.CreateVariables(net_desc); +scope.InitVariables(net_desc); + +// create a network according to `net_desc` +auto net = Net::CreateNet(net_desc); +// Add more operators if needed. +net->AddOp(add...); +net->AddOp(fc...); + +net->AddBackwardOps(); +net->AddOptimizerOps(); + +// run the network providing the `scope`. +net.Run(&scope); +``` + +## `NetBuilder` as a C++ syntax wrapper +This is a detailed description of the user-related C++ network API, and may not needed in the prototype development stage. + +The `NetBuilder` will give users a much simpler syntax as follows to create a network, and demonstrates how to use the `BaseNet`'s raw interfaces. + +```c++ +Variable* fc_out = builder.AddOp("fc", input=image, size=100, activation="Sigmoid"); +Variable* prediction = builder.AddOp("fc", input=fc_out, size=10, activation="Sigmoid"); +Variable* loss = builder.AddOp("cross_entropy", input=prediction, label=label); +Variable* avg_loss = builder.AddOp("mean", loss); + +builder.BackwardFrom(avg_loss) +builder.AddOptimization(1e-4, "adam"); +builder.Run(); +``` + +`NetBuilder` will call `Net` 's virtual functions to change the real network structure, here is a sample definition + +```c++ +class NetBuilder final { + public: + NetBuilder(Net* net) : net_(net) {} + + Variable* AddOp(const string& type, const vector& inputs, + size_t size, Activation act) { + // much code here. + // ... + net_->AddOp(def); + need_rebuild_net_ = true; + net_->InferShape(); + // ... + } + + Error BackwardFrom(const Variable& cost); + + Error Run(Scope* scope, OpContext* context, bool need_backward = true) { + // backward. + if (need_backward) { + if (need_rebuild_net_) { + AddBackwardOps(); + AddOptimizerOps(); + } + net_->Run(scope, context); + return; + } + // just forward. + net_->Run(scope, context, 0, last_forward_op_); + } + + protected: + Error AddBackwardOps(); + Error AddOptimizerOps(); + + private: + Net* net_; + OpIndex last_forward_op_{-1}; + bool need_rebuild_net_{true}; +} +``` + +## Compatibility with RNN + +Benefitting from the decoupling of `PlainNet.Run` and `Scope`, `PlainNet` is compatible with future RNN design, +for example we can implement a simple recurrent neural network as follows + +```c++ +// copy some `vars` form `source` to `target` +void Copy(const Scope &source, Scope &target, + const std::vector &vars); + +Scope default_scope; +// some initial mutations on `default_scope` here. + +auto rnn_step_net = PlainNet(rnn_step_net_def); + +// Create rnn's states, the last scope is used to store rnn outputs. +Scope *rnn_states = new Scope[num_states + 1]; + +for (int i = 0; i < num_states + 1; i++) { + // Initialize all rnn state scopes, copy parameters and so on. + rnn_states[i].CreateVars(rnn_step_net_def); + Copy(default_scope, rnn_states[i], rnn_related_vars); + // Prepare rnn's inlinks, just copy inlink variables to each state. + Copy(default_scope, rnn_states[i], inlink_vars); +} + +// Run the rnn. +for (int i = 0; i < num_states; i++) { + rnn_step_net.Run(rnn_states[i]); + // Copy current state's state variables to next state, the related variables + // are named like "previous_state_xxx". + Copy(rnn_states[i], rnn_states[i + 1], pre_state_vars) +} + +// Copy rnn's final outputs to `default_scope`. +Copy(rnn_states[num_states], default_scope, outlink_vars); +``` diff --git a/paddle/framework/op_desc.proto b/paddle/framework/op_desc.proto new file mode 100644 index 0000000000000000000000000000000000000000..89497f3c16bc28aa93b25a83c1f2eccafdf1c5b4 --- /dev/null +++ b/paddle/framework/op_desc.proto @@ -0,0 +1,56 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +syntax="proto2"; +package paddle.framework; + +import "attr_type.proto"; + +// AttrDesc is used to describe Attributes of an Operator. It contain's +// name, type, and value of Attribute. +// +// e.g, for scale=3.0: name=scala, type=AttrType.FLOAT, value=3.0 +message AttrDesc { + required string name = 1; + required AttrType type = 2; + optional int32 i = 3; + optional float f = 4; + optional string s = 5; + repeated int32 ints = 6; + repeated float floats = 7; + repeated string strings = 8; +}; + +// Protocol Message to describe an Operator. +// +// In PaddlePaddle, Operator is used to do a certain computation such +// as "add", "sub", "cosine", etc. +// (1) Operator needs to know the input and output variable names. +// (2) Some ops may have special attributes such as "scale" in "CosineOp". +// +// 3rd-party language can build this proto message and call +// AddOp(const OpDesc& op_desc) of Paddle core to create an Operator. +message OpDesc { + // input names of this Operator. + repeated string inputs = 1; + + // output names of this Operator. + repeated string outputs = 2; + + // type of this Operator, such as "add", "sub", "fc". + required string type = 3; + + // Attributes of this Operator. e.g., scale=3.0 in cosine op. + repeated AttrDesc attrs = 4; +}; \ No newline at end of file diff --git a/paddle/framework/op_desc_test.cc b/paddle/framework/op_desc_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..d0c52523b64725ee11c281b086f9ffed6a09e787 --- /dev/null +++ b/paddle/framework/op_desc_test.cc @@ -0,0 +1,35 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +TEST(OpDesc, Create) { + paddle::framework::OpDesc op_desc; + op_desc.set_type("add"); + op_desc.add_inputs("X"); + op_desc.add_inputs("Y"); + op_desc.add_outputs("Z"); + + auto attr = op_desc.mutable_attrs()->Add(); + attr->set_type(paddle::framework::AttrType::FLOAT); + attr->set_f(3.14); + + // required field name is not set, so IsInitialized should be false. + ASSERT_FALSE(op_desc.IsInitialized()); + + attr->set_name("add"); + // after all required fields are set, IsInitialized should be true now. + ASSERT_TRUE(op_desc.IsInitialized()); +} \ No newline at end of file diff --git a/paddle/framework/op_proto.proto b/paddle/framework/op_proto.proto new file mode 100644 index 0000000000000000000000000000000000000000..22df6f9c6b70277ddbf31e0432401889e3aa7483 --- /dev/null +++ b/paddle/framework/op_proto.proto @@ -0,0 +1,69 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +// Protocol Message for 3rd-party language binding. +// +// Paddle Python package will use `OpProto` to generate op creation methods. +// The op creation methods take user's input and generate `OpDesc` proto message, +// then pass `OpDesc` to C++ side and create Op pointer. +// +syntax="proto2"; +package paddle.framework; + +import "attr_type.proto"; + +// Attribute protocol message for 3rd-party language binding. +// It will store the Op support what attribute and what type. +message AttrProto { + // Supported attribute name. e.g. `scale` for cosine op. + required string name = 1; + + // Supported attribute type. + required AttrType type = 2; + + // Supported attribute comments. It helps 3rd-party language generate doc-string. + required string comment = 3; +} + +// Input or output message for 3rd-party language binding. +// It contains parameter name and its comments. +message VarProto { + // Input or output name in that op creation function. + // e.g. `cos(a, b, output, ...)`, "a", "b", "output" are names. + required string name = 1; + + // The comment for that input. It helps 3rd-party language generate doc-string. + required string comment = 2; +} + +// Op protocol message for 3rd-party language binding. +// It contains all information for generating op creation method. +message OpProto { + // The input information to generate op creation method. + repeated VarProto inputs = 1; + + // The output information to generate op creation method. + repeated VarProto outputs = 2; + + // The attribute information to generate op creation method. + repeated AttrProto attrs = 3; + + // The comments for that Op. It helps 3rd-party language generate + // doc-string. The whole documentation of that Op is generated by comment, + // inputs, outputs, attrs together. + required string comment = 4; + + // The type of that Op. + required string type = 5; +} diff --git a/paddle/framework/op_proto_test.cc b/paddle/framework/op_proto_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..9c054bde44e77571330cbc59074705f0cfc1cfb6 --- /dev/null +++ b/paddle/framework/op_proto_test.cc @@ -0,0 +1,31 @@ +#include +#include + +TEST(TestOpProto, ALL) { + paddle::framework::OpProto proto; + { + auto ipt = proto.mutable_inputs()->Add(); + *ipt->mutable_name() = "a"; + *ipt->mutable_comment() = "the one input of cosine op"; + } + { + auto ipt = proto.mutable_inputs()->Add(); + *ipt->mutable_name() = "b"; + *ipt->mutable_comment() = "the other input of cosine op"; + } + { + auto opt = proto.mutable_outputs()->Add(); + *opt->mutable_name() = "output"; + *opt->mutable_comment() = "the output of cosine op"; + } + { + auto attr = proto.mutable_attrs()->Add(); + *attr->mutable_name() = "scale"; + attr->set_type(paddle::framework::AttrType::FLOAT); + *attr->mutable_comment() = "the scale attribute of cosine op"; + } + proto.set_type("cos"); + *proto.mutable_comment() = "cosine op, output = scale * cos(a, b)"; + + ASSERT_TRUE(proto.IsInitialized()); +} \ No newline at end of file diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h new file mode 100644 index 0000000000000000000000000000000000000000..81241b5342d8900c205dd62f2a62dc2496010560 --- /dev/null +++ b/paddle/framework/op_registry.h @@ -0,0 +1,253 @@ +#pragma once + +#include "paddle/framework/attr_checker.h" + +//#include "paddle/framework/op_base.h" +#include "paddle/framework/op_desc.pb.h" +#include "paddle/framework/op_proto.pb.h" + +namespace paddle { +namespace framework { + +//==================For test================// +class OpBase { + public: + std::vector inputs_; + std::vector outputs_; + AttributeMap attr_map_; + + virtual std::string Run() const = 0; + virtual ~OpBase() {} +}; +//=========================================// + +// helper class to set attribute type +struct AttrTypeHelper { + template + static void SetAttrType(AttrProto* attr); + + static Attribute GetAttrValue(const AttrDesc& attr_desc) { + switch (attr_desc.type()) { + case paddle::framework::AttrType::INT: { + return attr_desc.i(); + } + case paddle::framework::AttrType::FLOAT: { + return attr_desc.f(); + } + case paddle::framework::AttrType::STRING: { + return attr_desc.s(); + } + case paddle::framework::AttrType::INTS: { + std::vector val(attr_desc.ints_size()); + for (int i = 0; i < attr_desc.ints_size(); ++i) { + val[i] = attr_desc.ints(i); + } + return val; + } + case paddle::framework::AttrType::FLOATS: { + std::vector val(attr_desc.floats_size()); + for (int i = 0; i < attr_desc.floats_size(); ++i) { + val[i] = attr_desc.floats(i); + } + return val; + } + case paddle::framework::AttrType::STRINGS: { + std::vector val(attr_desc.strings_size()); + for (int i = 0; i < attr_desc.strings_size(); ++i) { + val[i] = attr_desc.strings(i); + } + return val; + } + } + PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !"); + return boost::blank(); + } +}; + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::INT); +} + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::FLOAT); +} + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::STRING); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::INTS); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::FLOATS); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::STRINGS); +} + +// this class not only make proto but also init attribute checkers. +class OpProtoAndCheckerMaker { + public: + OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : proto_(proto), op_checker_(op_checker) {} + + protected: + void AddInput(const std::string& name, const std::string& comment) { + auto input = proto_->mutable_inputs()->Add(); + *(input->mutable_name()) = name; + *(input->mutable_comment()) = comment; + } + + void AddOutput(const std::string& name, const std::string& comment) { + auto output = proto_->mutable_outputs()->Add(); + *(output->mutable_name()) = name; + *(output->mutable_comment()) = comment; + } + + template + TypedAttrChecker& AddAttr(const std::string& name, + const std::string& comment) { + auto attr = proto_->mutable_attrs()->Add(); + *(attr->mutable_name()) = name; + *(attr->mutable_comment()) = comment; + AttrTypeHelper::SetAttrType(attr); + return op_checker_->AddAttrChecker(name); + } + + void AddType(const std::string& op_type) { proto_->set_type(op_type); } + + void AddComment(const std::string& comment) { + *(proto_->mutable_comment()) = comment; + } + + OpProto* proto_; + OpAttrChecker* op_checker_; +}; + +class OpRegistry { + typedef std::function OpCreator; + + public: + template + static void RegisterOp(const std::string& op_type) { + creators_[op_type] = []() { return new OpType; }; + OpProto& op_proto = protos_[op_type]; + OpAttrChecker& op_checker = op_checkers_[op_type]; + ProtoMakerType(&op_proto, &op_checker); + PADDLE_ENFORCE(op_proto.IsInitialized() == true, + "Fail to initialize %s's OpProto !", op_type); + } + + static OpBase* CreateOp(const OpDesc& op_desc) { + std::string op_type = op_desc.type(); + OpBase* op = (creators_.at(op_type))(); + (op->inputs_).resize(op_desc.inputs_size()); + for (int i = 0; i < op_desc.inputs_size(); ++i) { + (op->inputs_)[i] = op_desc.inputs(i); + } + (op->outputs_).resize(op_desc.outputs_size()); + for (int i = 0; i < op_desc.outputs_size(); ++i) { + (op->outputs_)[i] = op_desc.outputs(i); + } + for (int i = 0; i < op_desc.attrs_size(); ++i) { + const AttrDesc& ith_attr = op_desc.attrs(i); + std::string name = ith_attr.name(); + (op->attr_map_)[name] = AttrTypeHelper::GetAttrValue(ith_attr); + } + const OpAttrChecker& op_checker = op_checkers_.at(op_type); + op_checker.Check(op->attr_map_); + return op; + } + + private: + static std::unordered_map creators_; + static std::unordered_map protos_; + static std::unordered_map op_checkers_; +}; + +std::unordered_map> OpRegistry::creators_; +std::unordered_map OpRegistry::protos_; +std::unordered_map OpRegistry::op_checkers_; + +template +class OpRegisterHelper { + public: + OpRegisterHelper(std::string op_type) { + OpRegistry::RegisterOp(op_type); + } +}; + +#define REGISTER_OP(__op_class, __op_maker_class, __op_type) \ + class __op_class##Register { \ + private: \ + const static OpRegisterHelper<__op_class, __op_maker_class> reg; \ + }; \ + const OpRegisterHelper<__op_class, __op_maker_class> \ + __op_class##Register::reg(#__op_type); + +// Demos + +class CosineOp : public OpBase { + public: + virtual std::string Run() const { + std::string msg = "CosineOp runs! scale = " + + std::to_string(boost::get(attr_map_.at("scale"))); + return msg; + } +}; + +class CosineOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { + public: + CosineOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of cosine op"); + AddOutput("output", "output of cosine op"); + AddAttr("scale", "scale of cosine op") + .SetDefault(1.0) + .LargerThan(0.0); + AddType("cos"); + AddComment("This is cos op"); + } +}; + +REGISTER_OP(CosineOp, CosineOpProtoAndCheckerMaker, cos_sim) + +class MyTestOp : public OpBase { + public: + virtual std::string Run() const { + std::string msg = + "MyTestOp runs! test_attr = " + + std::to_string(boost::get(attr_map_.at("test_attr"))); + return msg; + } +}; + +class MyTestOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { + public: + MyTestOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of cosine op"); + AddOutput("output", "output of cosine op"); + auto my_checker = [](int i) { + PADDLE_ENFORCE(i % 2 == 0, "'test_attr' must be even!"); + }; + AddAttr("test_attr", "a simple test attribute") + .AddCustomChecker(my_checker); + AddType("my_test_op"); + AddComment("This is my_test op"); + } +}; + +REGISTER_OP(MyTestOp, MyTestOpProtoAndCheckerMaker, my_test_op) + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..17849ca0191db644884e766342b30461abf50298 --- /dev/null +++ b/paddle/framework/op_registry_test.cc @@ -0,0 +1,122 @@ +#include "paddle/framework/op_registry.h" +#include + +TEST(OpRegistry, CreateOp) { + paddle::framework::OpDesc op_desc; + op_desc.set_type("cos_sim"); + op_desc.add_inputs("aa"); + op_desc.add_outputs("bb"); + + auto attr = op_desc.mutable_attrs()->Add(); + attr->set_name("scale"); + attr->set_type(paddle::framework::AttrType::FLOAT); + attr->set_f(3.3); + + paddle::framework::OpBase* op = + paddle::framework::OpRegistry::CreateOp(op_desc); + std::string debug_str = op->Run(); + std::string str = "CosineOp runs! scale = " + std::to_string(3.3); + ASSERT_EQ(str.size(), debug_str.size()); + for (size_t i = 0; i < debug_str.length(); ++i) { + ASSERT_EQ(debug_str[i], str[i]); + } +} + +TEST(OpRegistry, IllegalAttr) { + paddle::framework::OpDesc op_desc; + op_desc.set_type("cos_sim"); + op_desc.add_inputs("aa"); + op_desc.add_outputs("bb"); + + auto attr = op_desc.mutable_attrs()->Add(); + attr->set_name("scale"); + attr->set_type(paddle::framework::AttrType::FLOAT); + attr->set_f(-2.0); + + bool caught = false; + try { + paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OpRegistry::CreateOp(op_desc); + } catch (paddle::framework::EnforceNotMet err) { + caught = true; + std::string msg = "larger_than check fail"; + const char* err_msg = err.what(); + for (size_t i = 0; i < msg.length(); ++i) { + ASSERT_EQ(err_msg[i], msg[i]); + } + } + ASSERT_TRUE(caught); +} + +TEST(OpRegistry, DefaultValue) { + paddle::framework::OpDesc op_desc; + op_desc.set_type("cos_sim"); + op_desc.add_inputs("aa"); + op_desc.add_outputs("bb"); + + paddle::framework::OpBase* op = + paddle::framework::OpRegistry::CreateOp(op_desc); + std::string debug_str = op->Run(); + float default_value = 1.0; + std::string str = "CosineOp runs! scale = " + std::to_string(default_value); + ASSERT_EQ(str.size(), debug_str.size()); + for (size_t i = 0; i < debug_str.length(); ++i) { + ASSERT_EQ(debug_str[i], str[i]); + } +} + +TEST(OpRegistry, CustomChecker) { + paddle::framework::OpDesc op_desc; + op_desc.set_type("my_test_op"); + op_desc.add_inputs("ii"); + op_desc.add_outputs("oo"); + + // attr 'test_attr' is not set + bool caught = false; + try { + paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OpRegistry::CreateOp(op_desc); + } catch (paddle::framework::EnforceNotMet err) { + caught = true; + std::string msg = "Attribute 'test_attr' is required!"; + const char* err_msg = err.what(); + for (size_t i = 0; i < msg.length(); ++i) { + ASSERT_EQ(err_msg[i], msg[i]); + } + } + ASSERT_TRUE(caught); + + // set 'test_attr' set to an illegal value + auto attr = op_desc.mutable_attrs()->Add(); + attr->set_name("test_attr"); + attr->set_type(paddle::framework::AttrType::INT); + attr->set_i(3); + caught = false; + try { + paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OpRegistry::CreateOp(op_desc); + } catch (paddle::framework::EnforceNotMet err) { + caught = true; + std::string msg = "'test_attr' must be even!"; + const char* err_msg = err.what(); + for (size_t i = 0; i < msg.length(); ++i) { + ASSERT_EQ(err_msg[i], msg[i]); + } + } + ASSERT_TRUE(caught); + + // set 'test_attr' set to a legal value + op_desc.mutable_attrs()->Clear(); + attr = op_desc.mutable_attrs()->Add(); + attr->set_name("test_attr"); + attr->set_type(paddle::framework::AttrType::INT); + attr->set_i(4); + paddle::framework::OpBase* op = + paddle::framework::OpRegistry::CreateOp(op_desc); + std::string debug_str = op->Run(); + std::string str = "MyTestOp runs! test_attr = " + std::to_string(4); + ASSERT_EQ(str.size(), debug_str.size()); + for (size_t i = 0; i < debug_str.length(); ++i) { + ASSERT_EQ(debug_str[i], str[i]); + } +} \ No newline at end of file diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h index 067f2a85264b462e96b65946b60b046172765a1d..ce5d98b04e6b53fcedc4fc4610d9390e64846b2a 100644 --- a/paddle/framework/tensor.h +++ b/paddle/framework/tensor.h @@ -14,33 +14,39 @@ limitations under the License. */ #pragma once +#include +#include +#include "paddle/framework/ddim.h" +#include "paddle/framework/enforce.h" +#include "paddle/memory/memory.h" +#include "paddle/platform/place.h" + namespace paddle { namespace framework { class Tensor { - using paddle::platform::Place; - using paddle::platform::get_place; - public: template const T* data() const { - PADDLE_ASSERT(holder_ != nullptr, - "Tensor::data must be called after Tensor::mutable_data"); - return static_cast(holder->Ptr()); + PADDLE_ENFORCE(holder_ != nullptr, + "Tensor::data must be called after Tensor::mutable_data."); + return static_cast(holder_->Ptr()); } template ::value>::type> - T* mutable_data(DDim dims, Place place) { - if (holder_ == nullptr || holder_->Place() != place || - holder_->Size() < dims.product() * sizeof(T)) { - holder_.reset(new PlaceholderImpl(place, dims.product() * sizeof(T))); + typename std::enable_if::value>::type* = nullptr> + T* mutable_data(DDim dims, paddle::platform::Place place) { + if (holder_ == nullptr || + !(holder_->Place() == + place) /* some versions of boost::variant don't have operator!= */ + || holder_->Size() < product(dims) * sizeof(T)) { + holder_.reset(new PlaceholderImpl(place, product(dims) * sizeof(T))); } return static_cast(holder_->Ptr()); } template ::value>::type> + typename std::enable_if::value>::type* = nullptr> T* mutable_data(DDim dims) { return mutable_data(dims, paddle::platform::get_place()); } @@ -51,27 +57,41 @@ class Tensor { struct Placeholder { virtual ~Placeholder() {} virtual void* Ptr() const = 0; - virtual Place Place() const = 0; + virtual paddle::platform::Place Place() const = 0; virtual size_t Size() const = 0; }; template struct PlaceholderImpl : public Placeholder { - PlaceholderImpl(Place pl, size_t size) - : ptr_(paddle::memory::Alloc(pl, size), paddle::memory::Deleter(pl)), - place_(pl), + private: + class Deleter { + public: + Deleter(platform::Place place) : place_(place) {} + void operator()(T* ptr) { + paddle::memory::Free(place_, static_cast(ptr)); + } + + private: + paddle::platform::Place place_; + }; + + public: + PlaceholderImpl(paddle::platform::Place place, size_t size) + : ptr_(static_cast(paddle::memory::Alloc(place, size)), + Deleter(place)), + place_(place), size_(size) {} virtual void* Ptr() const { return static_cast(ptr_.get()); } virtual size_t Size() const { return size_; } - virtual Place Place() const { return place_; } + virtual paddle::platform::Place Place() const { return place_; } - std::unique_ptr ptr_; - Place place_; // record the place of ptr_. - size_t size_; // size of the memory block. + std::unique_ptr ptr_; + paddle::platform::Place place_; // record the place of ptr_. + size_t size_; // size of the memory block. }; - std::unique_ptr holder_; // holds the memory block if allocated. + std::shared_ptr holder_; // holds the memory block if allocated. }; } // namespace framework diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..727d81f8d72e39ec564c42a48bf7ff64204adfff --- /dev/null +++ b/paddle/framework/tensor_test.cc @@ -0,0 +1,85 @@ +/* + Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "paddle/framework/tensor.h" +#include +#include + +TEST(Tensor, ASSERT) { + paddle::framework::Tensor cpu_tensor; + + bool caught = false; + try { + const double* p __attribute__((unused)) = cpu_tensor.data(); + } catch (paddle::framework::EnforceNotMet err) { + caught = true; + std::string msg = "Tensor::data must be called after Tensor::mutable_data."; + const char* what = err.what(); + for (size_t i = 0; i < msg.length(); ++i) { + ASSERT_EQ(what[i], msg[i]); + } + } + ASSERT_TRUE(caught); +} + +/* mutable_data() is not tested at present + because Memory::Alloc() and Memory::Free() have not been ready. + +TEST(Tensor, MutableData) { + using namespace paddle::framework; + using namespace paddle::platform; + { + Tensor cpu_tensor; + float* p1 = nullptr; + float* p2 = nullptr; + // initialization + p1 = cpu_tensor.mutable_data(make_ddim({1, 2, 3}), CPUPlace()); + EXPECT_NE(p1, nullptr); + // set cpu_tensor a new dim with large size + // momery is supposed to be re-allocated + p2 = cpu_tensor.mutable_data(make_ddim({3, 4})); + EXPECT_NE(p2, nullptr); + EXPECT_NE(p1, p2); + // set cpu_tensor a new dim with same size + // momery block is supposed to be unchanged + p1 = cpu_tensor.mutable_data(make_ddim({2, 2, 3})); + EXPECT_EQ(p1, p2); + // set cpu_tensor a new dim with smaller size + // momery block is supposed to be unchanged + p2 = cpu_tensor.mutable_data(make_ddim({2, 2})); + EXPECT_EQ(p1, p2); + } + + { + Tensor gpu_tensor; + float* p1 = nullptr; + float* p2 = nullptr; + // initialization + p1 = gpu_tensor.mutable_data(make_ddim({1, 2, 3}), GPUPlace()); + EXPECT_NE(p1, nullptr); + // set gpu_tensor a new dim with large size + // momery is supposed to be re-allocated + p2 = gpu_tensor.mutable_data(make_ddim({3, 4})); + EXPECT_NE(p2, nullptr); + EXPECT_NE(p1, p2); + // set gpu_tensor a new dim with same size + // momery block is supposed to be unchanged + p1 = gpu_tensor.mutable_data(make_ddim({2, 2, 3})); + EXPECT_EQ(p1, p2); + // set gpu_tensor a new dim with smaller size + // momery block is supposed to be unchanged + p2 = gpu_tensor.mutable_data(make_ddim({2, 2})); + EXPECT_EQ(p1, p2); + } +} +*/ diff --git a/paddle/gserver/layers/AverageLayer.h b/paddle/gserver/layers/AverageLayer.h index 332552a30479a368c24db10e5ef3a9d59408c8ef..db4a17bfb07de98fc092621a378c4fc23fa3adab 100644 --- a/paddle/gserver/layers/AverageLayer.h +++ b/paddle/gserver/layers/AverageLayer.h @@ -25,6 +25,10 @@ namespace paddle { * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = average_{for each instance in this sequence}{input[i]} + * If stride_ > 0: + * Output: a shorten sequence. Stride is the step size by which we slide a + * window upon the input sequence, and the average pooling + * operation is then applied to each interval independently. * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences diff --git a/paddle/gserver/layers/CrossChannelNormLayer.cpp b/paddle/gserver/layers/CrossChannelNormLayer.cpp index 3fbccc11032caa4878ce8dcfe7c34a261acee68b..d72503217f1c9533d0c78a2a1a853559f2a1294f 100644 --- a/paddle/gserver/layers/CrossChannelNormLayer.cpp +++ b/paddle/gserver/layers/CrossChannelNormLayer.cpp @@ -36,6 +36,16 @@ MatrixPtr CrossChannelNormLayer::createSpatialMatrix(MatrixPtr data, data->getData() + iter * spatialDim, 1, spatialDim, false, useGpu_); } +bool CrossChannelNormLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK(parameters_[0]); + const NormConfig& conf = config_.inputs(0).norm_conf(); + channels_ = conf.channels(); + scale_.reset(new Weight(channels_, 1, parameters_[0])); + return true; +} + void CrossChannelNormLayer::forward(PassType passType) { Layer::forward(passType); MatrixPtr inV = getInputValue(0); @@ -51,9 +61,7 @@ void CrossChannelNormLayer::forward(PassType passType) { Matrix::resizeOrCreate(dataBuffer_, batchSize, dataDim, false, useGpu_); Matrix::resizeOrCreate(spatialBuffer_, 1, spatialDim, false, useGpu_); Matrix::resizeOrCreate(normBuffer_, batchSize, spatialDim, false, useGpu_); - normBuffer_->zeroMem(); - // add eps to avoid overflow - normBuffer_->addScalar(*normBuffer_, 1e-6); + inV->square2(*dataBuffer_); for (size_t i = 0; i < batchSize; i++) { const MatrixPtr inVTmp = createSampleMatrix(inV, i, spatialDim); @@ -63,6 +71,8 @@ void CrossChannelNormLayer::forward(PassType passType) { // compute norm. spatialBuffer_->sumCols(*dataTmp, 1, 0); + // add eps to avoid overflow + spatialBuffer_->add(1e-6); spatialBuffer_->sqrt2(*spatialBuffer_); normTmp->copyFrom(*spatialBuffer_); outVTmp->copyFrom(*inVTmp); @@ -82,6 +92,9 @@ void CrossChannelNormLayer::backward(const UpdateCallback& callback) { size_t dataDim = inG->getWidth(); size_t spatialDim = dataDim / channels_; + MatrixPtr inGBuffer; + Matrix::resizeOrCreate(inGBuffer, channels_, spatialDim, false, useGpu_); + dataBuffer_->dotMul(*outG, *outV); Matrix::resizeOrCreate(scaleDiff_, channels_, 1, false, useGpu_); Matrix::resizeOrCreate(channelBuffer_, channels_, 1, false, useGpu_); @@ -100,22 +113,24 @@ void CrossChannelNormLayer::backward(const UpdateCallback& callback) { scaleDiff_->add(*channelBuffer_, 1.); sampleBuffer_->dotMul(*inVTmp, *outGTmp); - spatialBuffer_->sumCols(*sampleBuffer_, 1., 1.); + spatialBuffer_->sumCols(*sampleBuffer_, 1., 0.); // scale the grad - inGTmp->copyFrom(*inVTmp); - inGTmp->mulRowVector(*spatialBuffer_); + inGBuffer->copyFrom(*inVTmp); + inGBuffer->mulRowVector(*spatialBuffer_); // divide by square of norm spatialBuffer_->dotMul(*normTmp, *normTmp); - inGTmp->divRowVector(*spatialBuffer_); + inGBuffer->divRowVector(*spatialBuffer_); // subtract - inGTmp->add(*outGTmp, -1, 1); + inGBuffer->add(*outGTmp, -1, 1); // divide by norm - inGTmp->divRowVector(*normTmp); + inGBuffer->divRowVector(*normTmp); // scale the diff - inGTmp->mulColVector(*scale_->getW()); + inGBuffer->mulColVector(*scale_->getW()); + + inGTmp->add(*inGBuffer); } // updata scale - if (scale_->getWGrad()) scale_->getWGrad()->copyFrom(*scaleDiff_); + if (scale_->getWGrad()) scale_->getWGrad()->add(*scaleDiff_); scale_->getParameterPtr()->incUpdate(callback); } diff --git a/paddle/gserver/layers/DetectionOutputLayer.cpp b/paddle/gserver/layers/DetectionOutputLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8ab838e191314ab25469631626c0b0564d7fffda --- /dev/null +++ b/paddle/gserver/layers/DetectionOutputLayer.cpp @@ -0,0 +1,154 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "DetectionOutputLayer.h" + +namespace paddle { + +REGISTER_LAYER(detection_output, DetectionOutputLayer); + +bool DetectionOutputLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + auto& layerConf = config_.inputs(0).detection_output_conf(); + numClasses_ = layerConf.num_classes(); + inputNum_ = layerConf.input_num(); + nmsThreshold_ = layerConf.nms_threshold(); + confidenceThreshold_ = layerConf.confidence_threshold(); + nmsTopK_ = layerConf.nms_top_k(); + keepTopK_ = layerConf.keep_top_k(); + backgroundId_ = layerConf.background_id(); + return true; +} + +void DetectionOutputLayer::forward(PassType passType) { + Layer::forward(passType); + size_t batchSize = getInputValue(*getLocInputLayer(0))->getHeight(); + + locSizeSum_ = 0; + confSizeSum_ = 0; + for (size_t n = 0; n < inputNum_; ++n) { + const MatrixPtr inLoc = getInputValue(*getLocInputLayer(n)); + const MatrixPtr inConf = getInputValue(*getConfInputLayer(n)); + locSizeSum_ += inLoc->getElementCnt(); + confSizeSum_ += inConf->getElementCnt(); + } + + Matrix::resizeOrCreate(locTmpBuffer_, 1, locSizeSum_, false, useGpu_); + Matrix::resizeOrCreate( + confTmpBuffer_, confSizeSum_ / numClasses_, numClasses_, false, useGpu_); + + size_t locOffset = 0; + size_t confOffset = 0; + auto& layerConf = config_.inputs(0).detection_output_conf(); + for (size_t n = 0; n < inputNum_; ++n) { + const MatrixPtr inLoc = getInputValue(*getLocInputLayer(n)); + const MatrixPtr inConf = getInputValue(*getConfInputLayer(n)); + + size_t height = getInput(*getLocInputLayer(n)).getFrameHeight(); + if (!height) height = layerConf.height(); + size_t width = getInput(*getLocInputLayer(n)).getFrameWidth(); + if (!width) width = layerConf.width(); + locOffset += appendWithPermute(*inLoc, + height, + width, + locSizeSum_, + locOffset, + batchSize, + *locTmpBuffer_, + kNCHWToNHWC); + confOffset += appendWithPermute(*inConf, + height, + width, + confSizeSum_, + confOffset, + batchSize, + *confTmpBuffer_, + kNCHWToNHWC); + } + CHECK_EQ(locOffset, locSizeSum_ / batchSize); + CHECK_EQ(confOffset, confSizeSum_ / batchSize); + + MatrixPtr priorValue; + if (useGpu_) { + Matrix::resizeOrCreate(locCpuBuffer_, 1, locSizeSum_, false, false); + Matrix::resizeOrCreate( + confCpuBuffer_, confSizeSum_ / numClasses_, numClasses_, false, false); + MatrixPtr priorTmpValue = getInputValue(*getPriorBoxLayer()); + Matrix::resizeOrCreate( + priorCpuValue_, 1, priorTmpValue->getElementCnt(), false, false); + + locCpuBuffer_->copyFrom(*locTmpBuffer_); + confCpuBuffer_->copyFrom(*confTmpBuffer_); + priorCpuValue_->copyFrom(*priorTmpValue); + + locBuffer_ = locCpuBuffer_; + confBuffer_ = confCpuBuffer_; + priorValue = priorCpuValue_; + } else { + priorValue = getInputValue(*getPriorBoxLayer()); + locBuffer_ = locTmpBuffer_; + confBuffer_ = confTmpBuffer_; + } + confBuffer_->softmax(*confBuffer_); + + size_t numPriors = priorValue->getElementCnt() / 8; + std::vector> allDecodedBBoxes; + for (size_t n = 0; n < batchSize; ++n) { + std::vector decodedBBoxes; + for (size_t i = 0; i < numPriors; ++i) { + size_t priorOffset = i * 8; + size_t locPredOffset = n * numPriors * 4 + i * 4; + std::vector priorBBoxVec; + getBBoxFromPriorData( + priorValue->getData() + priorOffset, 1, priorBBoxVec); + std::vector> priorBBoxVar; + getBBoxVarFromPriorData( + priorValue->getData() + priorOffset, 1, priorBBoxVar); + std::vector locPredData; + for (size_t j = 0; j < 4; ++j) + locPredData.push_back(*(locBuffer_->getData() + locPredOffset + j)); + NormalizedBBox bbox = + decodeBBoxWithVar(priorBBoxVec[0], priorBBoxVar[0], locPredData); + decodedBBoxes.push_back(bbox); + } + allDecodedBBoxes.push_back(decodedBBoxes); + } + + std::vector>> allIndices; + size_t numKept = getDetectionIndices(confBuffer_->getData(), + numPriors, + numClasses_, + backgroundId_, + batchSize, + confidenceThreshold_, + nmsTopK_, + nmsThreshold_, + keepTopK_, + allDecodedBBoxes, + &allIndices); + + resetOutput(numKept, 7); + MatrixPtr outV = getOutputValue(); + getDetectionOutput(confBuffer_->getData(), + numKept, + numPriors, + numClasses_, + batchSize, + allIndices, + allDecodedBBoxes, + *outV); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/DetectionOutputLayer.h b/paddle/gserver/layers/DetectionOutputLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..a232af0a69141e238ae38c519a204cce25e9598d --- /dev/null +++ b/paddle/gserver/layers/DetectionOutputLayer.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include "DetectionUtil.h" +#include "Layer.h" + +namespace paddle { + +/** + * The detection output layer for a SSD detection task. This layer applies the + * Non-maximum suppression to the all predicted bounding box and keeps the + * Top-K bounding boxes. + * - Input: This layer needs three input layers: The first input layer + * is the priorbox layer. The rest two input layers are convolution + * layers for generating bbox location offset and the classification + * confidence. + * - Output: The predict bounding box locations. + */ + +class DetectionOutputLayer : public Layer { +public: + explicit DetectionOutputLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + + void backward(const UpdateCallback& callback = nullptr) {} + +protected: + inline LayerPtr getPriorBoxLayer() { return inputLayers_[0]; } + + inline LayerPtr getLocInputLayer(size_t index) { + return inputLayers_[1 + index]; + } + + inline LayerPtr getConfInputLayer(size_t index) { + return inputLayers_[1 + inputNum_ + index]; + } + +private: + size_t numClasses_; // number of classes + size_t inputNum_; // number of input layers + real nmsThreshold_; + real confidenceThreshold_; + size_t nmsTopK_; + size_t keepTopK_; + size_t backgroundId_; + + size_t locSizeSum_; + size_t confSizeSum_; + + MatrixPtr locBuffer_; + MatrixPtr confBuffer_; + MatrixPtr locTmpBuffer_; + MatrixPtr confTmpBuffer_; + MatrixPtr priorCpuValue_; + MatrixPtr locCpuBuffer_; + MatrixPtr confCpuBuffer_; +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/MaxLayer.h b/paddle/gserver/layers/MaxLayer.h index baa58ca2d7a6970f0d2f3ef6f8609404c82efa30..fa536fce2b4818337520478a6590bae36b26d09a 100644 --- a/paddle/gserver/layers/MaxLayer.h +++ b/paddle/gserver/layers/MaxLayer.h @@ -26,6 +26,10 @@ namespace paddle { * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = max_{for each instance in this sequence}{input[i]} + * If stride_ > 0: + * Output: a shorten sequence. Stride is the step size by which we slide a + * window upon the input sequence, and the max pooling operation is + * then applied to each interval independently. * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences diff --git a/paddle/gserver/layers/MultiBoxLossLayer.cpp b/paddle/gserver/layers/MultiBoxLossLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bbf1166dceddb1dbd672c42e8af22c113d2e3f54 --- /dev/null +++ b/paddle/gserver/layers/MultiBoxLossLayer.cpp @@ -0,0 +1,376 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MultiBoxLossLayer.h" +#include +#include +#include "DataLayer.h" + +namespace paddle { + +REGISTER_LAYER(multibox_loss, MultiBoxLossLayer); + +bool MultiBoxLossLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + + auto layerConf = config_.inputs(0).multibox_loss_conf(); + numClasses_ = layerConf.num_classes(); + inputNum_ = layerConf.input_num(); + overlapThreshold_ = layerConf.overlap_threshold(); + negPosRatio_ = layerConf.neg_pos_ratio(); + negOverlap_ = layerConf.neg_overlap(); + backgroundId_ = layerConf.background_id(); + return true; +} + +void MultiBoxLossLayer::forward(PassType passType) { + Layer::forward(passType); + size_t batchSize = getInputValue(*getLocInputLayer(0))->getHeight(); + resetOutput(batchSize, 1); + + // all location data and confidence score data + locSizeSum_ = 0; + confSizeSum_ = 0; + for (size_t n = 0; n < inputNum_; ++n) { + const MatrixPtr inLoc = getInputValue(*getLocInputLayer(n)); + const MatrixPtr inConf = getInputValue(*getConfInputLayer(n)); + locSizeSum_ += inLoc->getElementCnt(); + confSizeSum_ += inConf->getElementCnt(); + } + + // locBuffer layout: + // | xmin1 | ymin1 | xmax1 | ymax1 | xmin2 ...... + Matrix::resizeOrCreate(locTmpBuffer_, 1, locSizeSum_, false, useGpu_); + locBuffer_ = locTmpBuffer_; + + // confBuffer layout: + // | class1 score | class2 score | ... |classN score | class1 score | ...... + Matrix::resizeOrCreate(confTmpBuffer_, 1, confSizeSum_, false, useGpu_); + confBuffer_ = confTmpBuffer_; + + // concate location data and confidence score data + size_t locOffset = 0; + size_t confOffset = 0; + auto& layerConf = config_.inputs(0).multibox_loss_conf(); + for (size_t n = 0; n < inputNum_; ++n) { + const MatrixPtr inLoc = getInputValue(*getLocInputLayer(n)); + const MatrixPtr inConf = getInputValue(*getConfInputLayer(n)); + size_t height = getInput(*getLocInputLayer(n)).getFrameHeight(); + if (!height) height = layerConf.height(); + size_t width = getInput(*getLocInputLayer(n)).getFrameWidth(); + if (!width) width = layerConf.width(); + locOffset += appendWithPermute(*inLoc, + height, + width, + locSizeSum_, + locOffset, + batchSize, + *locBuffer_, + kNCHWToNHWC); + confOffset += appendWithPermute(*inConf, + height, + width, + confSizeSum_, + confOffset, + batchSize, + *confBuffer_, + kNCHWToNHWC); + } + CHECK_EQ(locOffset, locSizeSum_ / batchSize); + CHECK_EQ(confOffset, confSizeSum_ / batchSize); + + // priorValue layout: + // | xmin1 | ymin1 | xmax1 | ymax1 | xmin1Var | ymin1Var | xmax1Var | ymax1Var + // | xmin2 | ...... + MatrixPtr priorValue; + + // labelValue layout: + // | class1_1 | xmin1_1 | ymin1_1 | xmax1_1 | ymax1_1 | difficult1_1 | ...... + MatrixPtr labelValue; + + // Copy data from GPU to CPU if use GPU + if (useGpu_) { + Matrix::resizeOrCreate(locCpuBuffer_, 1, locSizeSum_, false, false); + Matrix::resizeOrCreate(confCpuBuffer_, 1, confSizeSum_, false, false); + MatrixPtr priorTmpValue = getInputValue(*getPriorBoxLayer()); + Matrix::resizeOrCreate( + priorCpuValue_, 1, priorTmpValue->getElementCnt(), false, false); + MatrixPtr labelTmpValue = getInputValue(*getLabelLayer()); + Matrix::resizeOrCreate(labelCpuValue_, + labelTmpValue->getHeight(), + labelTmpValue->getWidth(), + false, + false); + + locCpuBuffer_->copyFrom(*locTmpBuffer_); + confCpuBuffer_->copyFrom(*confTmpBuffer_); + priorCpuValue_->copyFrom(*priorTmpValue); + labelCpuValue_->copyFrom(*labelTmpValue); + + locBuffer_ = locCpuBuffer_; + confBuffer_ = confCpuBuffer_; + priorValue = priorCpuValue_; + labelValue = labelCpuValue_; + } else { + priorValue = getInputValue(*getPriorBoxLayer()); + labelValue = getInputValue(*getLabelLayer()); + } + + // Get max scores for each prior bbox. Used in negative mining + std::vector> allMaxConfScore; + numPriors_ = priorValue->getElementCnt() / 8; + getMaxConfidenceScores(confBuffer_->getData(), + batchSize, + numPriors_, + numClasses_, + backgroundId_, + &allMaxConfScore); + + // Match prior bbox to groundtruth bbox + Argument label = getInput(*getLabelLayer()); + const int* labelIndex = label.sequenceStartPositions->getData(false); + size_t seqNum = label.getNumSequences(); + numMatches_ = 0; + numNegs_ = 0; + allMatchIndices_.clear(); + allNegIndices_.clear(); + + std::pair retPair = generateMatchIndices(*priorValue, + numPriors_, + *labelValue, + labelIndex, + seqNum, + allMaxConfScore, + batchSize, + overlapThreshold_, + negOverlap_, + negPosRatio_, + &allMatchIndices_, + &allNegIndices_); + numMatches_ = retPair.first; + numNegs_ = retPair.second; + + // BBox location L1 smooth loss + locLoss_ = 0.0; + if (numMatches_ >= 1) { + size_t count = 0; + MatrixPtr locLossOutput; + Matrix::resizeOrCreate(locLossOutput, numMatches_ * 4, 1, false, false); + Matrix::resizeOrCreate(locGTData_, numMatches_ * 4, 1, false, false); + Matrix::resizeOrCreate(locDiff_, numMatches_ * 4, 1, false, false); + locDiff_->zeroMem(); + std::vector locGTData; + + real* locDiffData = locDiff_->getData(); + const real* locBufferData = locBuffer_->getData(); + for (size_t n = 0; n < batchSize; ++n) { + for (size_t i = 0; i < numPriors_; ++i) { + if (allMatchIndices_[n][i] == -1) continue; // match none + size_t locOffset = + n * (locBuffer_->getElementCnt() / batchSize) + i * 4; + std::copy(locBufferData + locOffset, + locBufferData + locOffset + 4, + locDiffData + count); + count += 4; + const int gtIdx = allMatchIndices_[n][i]; + size_t priorOffset = i * 8; + std::vector priorBBoxVec; + getBBoxFromPriorData( + priorValue->getData() + priorOffset, 1, priorBBoxVec); + std::vector> priorBBoxVar; + getBBoxVarFromPriorData( + priorValue->getData() + priorOffset, 1, priorBBoxVar); + size_t labelOffset = (labelIndex[n] + gtIdx) * 6; + std::vector gtBBoxVec; + getBBoxFromLabelData(labelValue->getData() + labelOffset, 1, gtBBoxVec); + std::vector gtEncode; + encodeBBoxWithVar( + priorBBoxVec[0], priorBBoxVar[0], gtBBoxVec[0], gtEncode); + locGTData.insert(locGTData.end(), gtEncode.begin(), gtEncode.end()); + } + } + locGTData_->copyFrom(&locGTData[0], numMatches_ * 4); + locLossOutput->smoothL1(*locDiff_, *locGTData_, 0.0); + locLoss_ = locLossOutput->getSum() / numMatches_; + } + + // BBox confidence softmax loss + confLoss_ = 0; + numConf_ = numMatches_ + numNegs_; + if (numConf_ >= 1) { + Matrix::resizeOrCreate(confProb_, numConf_, numClasses_, false, false); + IVector::resizeOrCreate(confGTData_, numConf_, false); + confProb_->zeroMem(); + size_t count = 0; + + std::vector confPredData; + real* confProbData = confProb_->getData(); + const real* confBufferData = confBuffer_->getData(); + for (size_t n = 0; n < batchSize; ++n) { + for (size_t i = 0; i < numPriors_; ++i) { + if (allMatchIndices_[n][i] == -1) continue; + size_t labelOffset = (labelIndex[n] + allMatchIndices_[n][i]) * 6; + const int gtLabel = (labelValue->getData() + labelOffset)[0]; + confGTData_->getData()[count] = gtLabel; + size_t confOffset = n * numPriors_ * numClasses_ + i * numClasses_; + std::copy(confBufferData + confOffset, + confBufferData + confOffset + numClasses_, + confProbData + count * numClasses_); + confPredData.reserve(confPredData.size() + numClasses_); + confPredData.insert(confPredData.end(), + confBufferData + confOffset, + confBufferData + confOffset + numClasses_); + ++count; + } + // Negative mining samples + for (size_t i = 0; i < allNegIndices_[n].size(); ++i) { + confGTData_->getData()[count] = backgroundId_; + size_t confOffset = + n * numPriors_ * numClasses_ + allNegIndices_[n][i] * numClasses_; + std::copy(confBufferData + confOffset, + confBufferData + confOffset + numClasses_, + confProbData + count * numClasses_); + confPredData.reserve(confPredData.size() + numClasses_); + confPredData.insert(confPredData.end(), + confBufferData + confOffset, + confBufferData + confOffset + numClasses_); + ++count; + } + } + CHECK_EQ(numConf_, count); + confProb_->softmax(*confProb_); + MatrixPtr confLossOutput; + Matrix::resizeOrCreate(confLossOutput, numConf_, 1, false, false); + confLossOutput->oneHotCrossEntropy(*confProb_, *confGTData_); + confLoss_ = confLossOutput->getSum() / numMatches_; + } + real loss = locLoss_ + confLoss_; + MatrixPtr outV = getOutputValue(); + outV->assign(loss); +} + +void MultiBoxLossLayer::backward(const UpdateCallback& callback) { + size_t batchSize = getInputValue(*getLocInputLayer(0))->getHeight(); + locBuffer_->zeroMem(); + confBuffer_->zeroMem(); + + // Back propagate on location prediction + if (numMatches_ >= 1) { + MatrixPtr locDiffBuffer; + Matrix::resizeOrCreate(locDiffBuffer, numMatches_ * 4, 1, false, false); + locDiffBuffer->smoothL1Bp(*locDiff_, *locGTData_, 0.0); + locDiff_->copyFrom(*locDiffBuffer); + // scale gradient + for (size_t i = 0; i < numMatches_ * 4; ++i) + locDiff_->getData()[i] *= (1. / numMatches_); + // Copy gradient back + size_t count = 0; + const real* locDiffData = locDiff_->getData(); + for (size_t n = 0; n < batchSize; ++n) { + for (size_t i = 0; i < numPriors_; ++i) { + if (allMatchIndices_[n][i] == -1) continue; + real* locBufferData = + locBuffer_->getData() + n * numPriors_ * 4 + i * 4; + std::copy(locDiffData + count * 4, + locDiffData + (count + 1) * 4, + locBufferData); + ++count; + } + } + CHECK_EQ(count, numMatches_); + } + + if (numConf_ >= 1) { + for (size_t i = 0; i < numConf_; ++i) + confProb_->getData()[i * numClasses_ + confGTData_->getData()[i]] -= 1; + for (size_t i = 0; i < numConf_ * numClasses_; ++i) + confProb_->getData()[i] *= (1. / numMatches_); + size_t count = 0; + const real* confProbData = confProb_->getData(); + for (size_t n = 0; n < batchSize; ++n) { + for (size_t i = 0; i < numPriors_; ++i) { + if (allMatchIndices_[n][i] == -1) continue; + real* confDiffData = confBuffer_->getData() + + n * numPriors_ * numClasses_ + i * numClasses_; + std::copy(confProbData + count * numClasses_, + confProbData + (count + 1) * numClasses_, + confDiffData); + ++count; + } + for (size_t i = 0; i < allNegIndices_[n].size(); ++i) { + int idx = allNegIndices_[n][i]; + real* confDiffData = confBuffer_->getData() + + n * numPriors_ * numClasses_ + idx * numClasses_; + std::copy(confProbData + count * numClasses_, + confProbData + (count + 1) * numClasses_, + confDiffData); + ++count; + } + } + CHECK_EQ(count, numConf_); + } + if (useGpu_) { + locTmpBuffer_->copyFrom(*locCpuBuffer_); + confTmpBuffer_->copyFrom(*confCpuBuffer_); + locBuffer_ = locTmpBuffer_; + confBuffer_ = confTmpBuffer_; + } + // copy back + size_t locOffset = 0; + size_t confOffset = 0; + auto layerConf = config_.inputs(0).multibox_loss_conf(); + for (size_t n = 0; n < inputNum_; ++n) { + const MatrixPtr inLocG = getInputGrad(*getLocInputLayer(n)); + const MatrixPtr inConfG = getInputGrad(*getConfInputLayer(n)); + size_t height = getInput(*getLocInputLayer(n)).getFrameHeight(); + // only for unittest, there are no width and height information + // when constructing matrix in unittest, so we should + // set the shape in configuration + if (!height) height = layerConf.height(); + size_t width = getInput(*getLocInputLayer(n)).getFrameWidth(); + if (!width) width = layerConf.width(); + + // NHWC to NCHW + MatrixPtr locGBuffer; + Matrix::resizeOrCreate( + locGBuffer, inLocG->getHeight(), inLocG->getWidth(), false, useGpu_); + MatrixPtr confGBuffer; + Matrix::resizeOrCreate( + confGBuffer, inConfG->getHeight(), inConfG->getWidth(), false, useGpu_); + + locOffset += decomposeWithPermute(*locBuffer_, + height, + width, + locSizeSum_, + locOffset, + batchSize, + *locGBuffer, + kNHWCToNCHW); + inLocG->add(*locGBuffer); + confOffset += decomposeWithPermute(*confBuffer_, + height, + width, + confSizeSum_, + confOffset, + batchSize, + *confGBuffer, + kNHWCToNCHW); + inConfG->add(*confGBuffer); + } + CHECK_EQ(locOffset, locSizeSum_ / batchSize); + CHECK_EQ(confOffset, confSizeSum_ / batchSize); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MultiBoxLossLayer.h b/paddle/gserver/layers/MultiBoxLossLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..9935da56446c1508549906becfd28548d5deecde --- /dev/null +++ b/paddle/gserver/layers/MultiBoxLossLayer.h @@ -0,0 +1,103 @@ +/* copyright (c) 2016 paddlepaddle authors. all rights reserve. + +licensed under the apache license, version 2.0 (the "license"); +you may not use this file except in compliance with the license. +you may obtain a copy of the license at + + http://www.apache.org/licenses/license-2.0 + +unless required by applicable law or agreed to in writing, software +distributed under the license is distributed on an "as is" basis, +without warranties or conditions of any kind, either express or implied. +see the license for the specific language governing permissions and +limitations under the license. */ + +#pragma once + +#include +#include "CostLayer.h" +#include "DataLayer.h" +#include "DetectionUtil.h" +#include "Layer.h" + +using std::vector; +using std::pair; + +namespace paddle { + +/** + * The multibox loss layer for a SSD detection task. + * The loss is composed by the location loss and the confidence loss. + * The location loss is a smooth L1 loss and the confidence loss is + * a softmax loss. + * - Input: This layer needs four input layers: The first input layer + * is the priorbox layer and the second layer is a label layer. + * The rest two input layers are convolution layers for generating + * bbox location offset and the classification confidence. + * - Output: The Single Shot Multibox Detection loss value. + * Reference: + * Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, Scott Reed, + * Cheng-Yang Fu, Alexander C. Berg. SSD: Single Shot MultiBox Detector + */ + +class MultiBoxLossLayer : public CostLayer { +public: + explicit MultiBoxLossLayer(const LayerConfig& config) : CostLayer(config) {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + + void backward(const UpdateCallback& callback = nullptr); + + void forwardImp(Matrix& output, Argument& label, Matrix& cost) {} + + void backwardImp(Matrix& outputValue, Argument& label, Matrix& outputGrad) {} + +protected: + inline LayerPtr getPriorBoxLayer() { return inputLayers_[0]; } + inline LayerPtr getLabelLayer() { return inputLayers_[1]; } + inline LayerPtr getLocInputLayer(size_t index) { + return inputLayers_[2 + index]; + } + inline LayerPtr getConfInputLayer(size_t index) { + return inputLayers_[2 + inputNum_ + index]; + } + +protected: + size_t numClasses_; + real overlapThreshold_; + real negPosRatio_; + real negOverlap_; + size_t inputNum_; + size_t backgroundId_; + + real locLoss_; + real confLoss_; + + size_t numPriors_; + size_t numMatches_; + size_t numNegs_; + size_t numConf_; + size_t locSizeSum_; + size_t confSizeSum_; + + vector> allMatchIndices_; + vector> allNegIndices_; + MatrixPtr locGTData_; + IVectorPtr confGTData_; + + MatrixPtr locBuffer_; + MatrixPtr confBuffer_; + MatrixPtr locDiff_; + MatrixPtr confProb_; + + MatrixPtr labelCpuValue_; + MatrixPtr priorCpuValue_; + MatrixPtr locCpuBuffer_; + MatrixPtr confCpuBuffer_; + MatrixPtr locTmpBuffer_; + MatrixPtr confTmpBuffer_; +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/NormLayer.cpp b/paddle/gserver/layers/NormLayer.cpp index e094078bfe86e30c06e1b80ebc04c8213fe9abcf..caef7100929c7e3df4904b577cb7c2178466ddfc 100644 --- a/paddle/gserver/layers/NormLayer.cpp +++ b/paddle/gserver/layers/NormLayer.cpp @@ -56,14 +56,4 @@ bool ResponseNormLayer::init(const LayerMap& layerMap, return true; } -bool CrossChannelNormLayer::init(const LayerMap& layerMap, - const ParameterMap& parameterMap) { - Layer::init(layerMap, parameterMap); - CHECK(parameters_[0]); - const NormConfig& conf = config_.inputs(0).norm_conf(); - channels_ = conf.channels(); - scale_.reset(new Weight(channels_, 1, parameters_[0])); - return true; -} - } // namespace paddle diff --git a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp index 944c7051668dccf39dd2ace14986d43c8a14e452..323cc47df199a6cb5e8e87cad4aaf51a92c36f81 100644 --- a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp +++ b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp @@ -26,10 +26,9 @@ namespace paddle { * If SequenceLevel = kNonseq: * Output: a sequence containing only the last instance of the input sequence * If stride_ > 0: - * Output: a shorten sequence. The operation of getting last instance of a - * sequence is independently performed on every slice of the input - * sequence, which is obtained by sliding a window with the window - * size set to stride_. + * Output: a shorten sequence. Stride is the step size by which we slide a + * window upon the input sequence, and getting last instance + * operation is then applied to each interval independently. * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: a sequence containing only the last instance of each sub-sequence @@ -73,8 +72,7 @@ bool SequenceLastInstanceLayer::init(const LayerMap& layerMap, void SequenceLastInstanceLayer::forward(PassType passType) { SequencePoolLayer::forward(passType); - auto starts = (stride_ > 0) ? stridePositions_->getData() - : startPositions_->getData(false); + auto starts = startPositions_->getData(false); MatrixPtr inputValue = getInputValue(0); MatrixPtr outputValue = getOutputValue(); diff --git a/paddle/gserver/layers/SequencePoolLayer.cpp b/paddle/gserver/layers/SequencePoolLayer.cpp index 4179a9e7e0cb58fcb49bff712e62b9f3fea373bd..2a693b110a562ce3938643c919bfb1a4d3cd1f80 100644 --- a/paddle/gserver/layers/SequencePoolLayer.cpp +++ b/paddle/gserver/layers/SequencePoolLayer.cpp @@ -72,9 +72,8 @@ void SequencePoolLayer::forward(PassType passType) { if (stride_ > 0) { CHECK_EQ(input.hasSubseq(), 0UL) << "sequence stride pooling is invalid for hasSubseq now"; - output_.poolSequenceWithStride( - input, stride_, &stridePositions_, reversed_); - newBatchSize_ = stridePositions_->getSize() - 1; + output_.poolSequenceWithStride(input, stride_, &startPositions_, reversed_); + newBatchSize_ = startPositions_->getSize() - 1; } resetOutput(newBatchSize_, dim); diff --git a/paddle/gserver/layers/SequencePoolLayer.h b/paddle/gserver/layers/SequencePoolLayer.h index 293d1bf27823ffb0ebddba95461883d646f159ae..e207afd1dce80e646b220c5be601fd3a6bd36bac 100644 --- a/paddle/gserver/layers/SequencePoolLayer.h +++ b/paddle/gserver/layers/SequencePoolLayer.h @@ -28,8 +28,9 @@ namespace paddle { * sequence}{input[i]} * If stride_ > 0: * Check input sequence must not have sub-sequence - * Output: a shorten sequence, pooling is performed upon a small local - * area + * Output: a shorten sequence. Stride is the step size by which we slide + * a window upon the input sequence, and the pooling operation + * is then applied to each interval independently. * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences @@ -47,8 +48,6 @@ protected: size_t newBatchSize_; ICpuGpuVectorPtr startPositions_; int stride_; - // Store the start position of each window. - IVectorPtr stridePositions_; // Whether the input sequence is reversed or not. bool reversed_ = false; diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 3c4128b5b8a0ea420bd3027b9a36e5f75087c3cb..92f6cbcfe5a0e23c5939b1689a3e339367450387 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -45,6 +45,13 @@ add_unittest_without_exec(test_PriorBox add_test(NAME test_PriorBox COMMAND test_PriorBox) +################# test_DetectionOutput ####################### +add_unittest_without_exec(test_DetectionOutput + test_DetectionOutput.cpp + LayerGradUtil.cpp) + +add_test(NAME test_DetectionOutput + COMMAND test_DetectionOutput) ################# test_ConvUnify ####################### add_unittest_without_exec(test_ConvUnify test_ConvUnify.cpp diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index a0b1cd471dd02fd20bb2247395bdb74651610bbf..9eca58f1a1baa6fb1c404a91a345bc7f9d6b4acc 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -241,7 +241,7 @@ void testBatchState(LayerPtr testLayer, std::vector args; args.push_back(out); - EXPECT_EQ(0, Argument::sum(args)) << "testBatchState failed"; + ASSERT_NEAR(0, Argument::sum(args), 1e-5) << "testBatchState failed"; for (size_t seqId = 0; seqId < numSequences; ++seqId) { start[seqId] += seqLens[seqId]; } @@ -387,6 +387,31 @@ void initDataLayer(TestConfig testConf, data.value->sigmoid(*data.value); data.grad->zeroMem(); break; + case INPUT_SELF_DEFINE_DATA: { + size_t height = testConf.inputDefs[i].selfDefinedData->getHeight(); + size_t width = testConf.inputDefs[i].selfDefinedData->getWidth(); + CHECK_GT(static_cast(height), 0); + CHECK_GT(static_cast(width), 0); + data.value = Matrix::create(height, width, false, useGpu); + data.grad = Matrix::create(height, width, false, useGpu); + data.value->copyFrom(*testConf.inputDefs[i].selfDefinedData); + data.grad->zeroMem(); + + const std::vector& labelSeqStartPositions = + testConf.inputDefs[i].labelSeqStartPositions; + if (labelSeqStartPositions.size() != 0) { + CHECK(!sequenceStartPositions); + CHECK_GE(static_cast(labelSeqStartPositions.size()), 2); + + sequenceStartPositions = + ICpuGpuVector::create(labelSeqStartPositions.size(), useGpu); + sequenceStartPositions->copyFrom(labelSeqStartPositions.data(), + labelSeqStartPositions.size(), + useGpu); + data.sequenceStartPositions = sequenceStartPositions; + } + break; + } default: LOG(FATAL) << " unknown inputType "; return; @@ -440,7 +465,6 @@ void initTestLayer(TestConfig testConf, ParameterConfig paraConfig) { paraConfig.set_name(paraName); paraConfig.set_size(paraSize); - paraConfig.set_initial_std(1); paraConfig.set_is_static(isStatic); auto para = std::make_shared(paraConfig, FLAGS_use_gpu, initialize); @@ -474,6 +498,9 @@ void initTestLayer(TestConfig testConf, paraConfig.add_dims((*layerMap)[input.input_layer_name()]->getSize()); paraConfig.add_dims(testConf.layerConfig.size()); } + CHECK_GE(testConf.paramInitialStd, 0); + paraConfig.set_initial_mean(testConf.paramInitialMean); + paraConfig.set_initial_std(testConf.paramInitialStd); initParameter(paraName, paraSize, inputDef.isStatic, false, paraConfig); } } diff --git a/paddle/gserver/tests/LayerGradUtil.h b/paddle/gserver/tests/LayerGradUtil.h index 9f68eb64d0b4ad27306d3b20387d74a7e438d910..d299b4dd09418589514d99a72f83e1103ace7de1 100644 --- a/paddle/gserver/tests/LayerGradUtil.h +++ b/paddle/gserver/tests/LayerGradUtil.h @@ -31,7 +31,8 @@ enum InputType { INPUT_SEQUENCE_LABEL, INPUT_SPARSE_NON_VALUE_DATA, INPUT_SPARSE_FLOAT_VALUE_DATA, - INPUT_DENSE_DIM_DATA, // using sequence length to init dense data + INPUT_DENSE_DIM_DATA, // using sequence length to init dense data + INPUT_SELF_DEFINE_DATA, // support customizing for input value }; struct ParaSparse { @@ -66,6 +67,7 @@ struct InputDef { bool isStatic; std::vector labelInitValue; std::vector labelSeqStartPositions; + MatrixPtr selfDefinedData; InputDef(InputType type, string nameIn, size_t dimIn, size_t sizeIn) { inputType = type; @@ -76,6 +78,20 @@ struct InputDef { isStatic = false; } + InputDef(InputType type, + string nameIn, + MatrixPtr selfDefinedData, + std::vector selfDefinedSeqStartPos = {}) + : labelSeqStartPositions(selfDefinedSeqStartPos), + selfDefinedData(selfDefinedData) { + inputType = type; + name = nameIn; + dim = 0; + sparse = {""}; + paraSize = 0; + isStatic = false; + } + InputDef(InputType type, string nameIn, size_t dimIn, @@ -109,12 +125,16 @@ struct TestConfig { LayerConfig layerConfig; std::vector inputDefs; size_t biasSize; + real paramInitialMean; + real paramInitialStd; bool testAccumulate; bool testState; bool staticBias; bool testBatchState; TestConfig() : biasSize(0), + paramInitialMean(0.0), + paramInitialStd(1.0), testAccumulate(true), testState(false), staticBias(false), diff --git a/paddle/gserver/tests/test_DetectionOutput.cpp b/paddle/gserver/tests/test_DetectionOutput.cpp new file mode 100644 index 0000000000000000000000000000000000000000..af43dc51fad35c834635b543b1a016f6d717de1e --- /dev/null +++ b/paddle/gserver/tests/test_DetectionOutput.cpp @@ -0,0 +1,194 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include + +#include "LayerGradUtil.h" +#include "paddle/testing/TestUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +// Do one forward pass of priorBox layer and check to see if its output +// matches the given result +void doOneDetectionOutputTest(MatrixPtr& inputLoc, + MatrixPtr& inputConf, + MatrixPtr& inputPriorBox, + size_t feature_map_width, + size_t feature_map_height, + real nms_threshold, + bool use_gpu, + MatrixPtr& result) { + // Setting up the detection output layer + TestConfig configt; + configt.layerConfig.set_type("detection_output"); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + configt.layerConfig.add_inputs(); + configt.layerConfig.add_inputs(); + + DetectionOutputConfig* detOutput = input->mutable_detection_output_conf(); + detOutput->set_width(feature_map_width); + detOutput->set_height(feature_map_height); + detOutput->set_nms_threshold(nms_threshold); + detOutput->set_num_classes(2); + detOutput->set_nms_top_k(20); + detOutput->set_keep_top_k(10); + detOutput->set_background_id(0); + detOutput->set_confidence_threshold(0.01); + detOutput->set_input_num(1); + configt.inputDefs.push_back({INPUT_DATA_TARGET, "priorbox", 32, 0}); + configt.inputDefs.push_back({INPUT_DATA, "input_loc", 16, 0}); + configt.inputDefs.push_back({INPUT_DATA, "input_conf", 8, 0}); + + // data layer initialize + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer( + configt, &dataLayers, &datas, &layerMap, "priorbox", 1, false, use_gpu); + + dataLayers[0]->getOutputValue()->copyFrom(*inputPriorBox); + dataLayers[1]->getOutputValue()->copyFrom(*inputLoc); + dataLayers[2]->getOutputValue()->copyFrom(*inputConf); + + // test layer initialize + bool store_FLAGS_use_gpu = FLAGS_use_gpu; + FLAGS_use_gpu = use_gpu; + std::vector parameters; + LayerPtr detectionOutputLayer; + initTestLayer(configt, &layerMap, ¶meters, &detectionOutputLayer); + FLAGS_use_gpu = store_FLAGS_use_gpu; + detectionOutputLayer->forward(PASS_GC); + checkMatrixEqual(detectionOutputLayer->getOutputValue(), result); +} + +TEST(Layer, detectionOutputLayerFwd) { + bool useGpu = false; + // CPU case 1. + MatrixPtr inputLoc; + MatrixPtr inputConf; + MatrixPtr inputPriorBox; + MatrixPtr result, result2, result3, result4; + real nmsTreshold = 0.01; + real inputLocData[] = {0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1, + 0.1}; + real inputConfData[] = {0.1, 0.9, 0.2, 0.8, 0.3, 0.7, 0.4, 0.6}; + real inputPriorBoxData[] = {0.1, 0.1, 0.5, 0.5, 0.1, 0.1, 0.2, 0.2, + 0.2, 0.2, 0.6, 0.6, 0.1, 0.1, 0.2, 0.2, + 0.3, 0.3, 0.7, 0.7, 0.1, 0.1, 0.2, 0.2, + 0.4, 0.4, 0.8, 0.8, 0.1, 0.1, 0.2, 0.2}; + real resultData[] = { + 0, 1, 0.68997443, 0.099959746, 0.099959746, 0.50804031, 0.50804031}; + inputLoc = Matrix::create(1, 16, false, useGpu); + inputConf = Matrix::create(1, 8, false, useGpu); + inputPriorBox = Matrix::create(1, 32, false, useGpu); + result = Matrix::create(1, 7, false, useGpu); + inputLoc->setData(inputLocData); + inputConf->setData(inputConfData); + inputPriorBox->setData(inputPriorBoxData); + result->setData(resultData); + doOneDetectionOutputTest(inputLoc, + inputConf, + inputPriorBox, + /* feature_map_width */ 1, + /* feature_map_height */ 1, + nmsTreshold, + useGpu, + result); + + // CPU case 2. + nmsTreshold = 0.2; + result2 = Matrix::create(2, 7, false, useGpu); + real resultData2[] = {0, + 1, + 0.68997443, + 0.099959746, + 0.099959746, + 0.50804031, + 0.50804031, + 0, + 1, + 0.59868765, + 0.29995975, + 0.29995975, + 0.70804024, + 0.70804024}; + result2->setData(resultData2); + doOneDetectionOutputTest(inputLoc, + inputConf, + inputPriorBox, + /* feature_map_width */ 1, + /* feature_map_height */ 1, + nmsTreshold, + useGpu, + result2); + +#ifndef PADDLE_ONLY_CPU + // GPU case 1. + useGpu = true; + inputLoc = Matrix::create(1, 16, false, useGpu); + inputConf = Matrix::create(1, 8, false, useGpu); + inputPriorBox = Matrix::create(1, 32, false, useGpu); + inputLoc->copyFrom(inputLocData, 16); + inputConf->copyFrom(inputConfData, 8); + inputPriorBox->copyFrom(inputPriorBoxData, 32); + + nmsTreshold = 0.01; + result3 = Matrix::create(1, 7, false, useGpu); + result3->copyFrom(resultData, 7); + doOneDetectionOutputTest(inputLoc, + inputConf, + inputPriorBox, + /* feature_map_width */ 1, + /* feature_map_height */ 1, + nmsTreshold, + useGpu, + result3); + + // GPU case 2. + nmsTreshold = 0.2; + result4 = Matrix::create(2, 7, false, useGpu); + result4->copyFrom(resultData2, 14); + doOneDetectionOutputTest(inputLoc, + inputConf, + inputPriorBox, + /* feature_map_width */ 1, + /* feature_map_height */ 1, + nmsTreshold, + useGpu, + result4); +#endif +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 297756025bcad79d49ec321414ed2e91f1c0758a..67251f08e34faff57d9e6fd6a1163ba655619a8b 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -845,8 +845,12 @@ void testDegradeLayer(bool hasSubseq, TEST(Layer, MaxLayer) { testDegradeLayer(false, "max", "non-seq", -1); // seq max to non-seq - testDegradeLayer(true, "max", "non-seq", -1); // hasSubseq max to non-seq - testDegradeLayer(true, "max", "seq", -1); // hasSubseq max to seq + testDegradeLayer(false, + "max", + "non-seq", + 5); // seq max to a shorten seq, stride window = 5 + testDegradeLayer(true, "max", "non-seq", -1); // hasSubseq max to non-seq + testDegradeLayer(true, "max", "seq", -1); // hasSubseq max to seq } TEST(Layer, SequenceLastInstanceLayer) { @@ -868,6 +872,10 @@ TEST(Layer, SequenceLastInstanceLayer) { TEST(Layer, AverageLayer) { testDegradeLayer(false, "average", "non-seq", -1); // seq average to non-seq + testDegradeLayer(false, + "average", + "non-seq", + 5); // seq average to a shorten seq, stride window = 5 testDegradeLayer( true, "average", "non-seq", -1); // hasSubseq average to non-seq testDegradeLayer(true, "average", "seq", -1); // hasSubseq average to seq @@ -1661,6 +1669,8 @@ TEST(Layer, PadLayer) { TEST(Layer, CrossChannelNormLayer) { TestConfig config; + config.paramInitialMean = 1.; + config.paramInitialStd = 0.; config.layerConfig.set_type("norm"); config.layerConfig.set_size(100); LayerInputConfig* input = config.layerConfig.add_inputs(); @@ -1674,7 +1684,7 @@ TEST(Layer, CrossChannelNormLayer) { config.inputDefs.push_back({INPUT_DATA, "layer_0", 100, 10}); for (auto useGpu : {false, true}) { - testLayerGrad(config, "cross-channel-norm", 10, false, useGpu, false, 5); + testLayerGrad(config, "cross-channel-norm", 10, false, useGpu, false); } } @@ -1692,6 +1702,70 @@ TEST(Layer, smooth_l1) { } } +TEST(Layer, multibox_loss) { + TestConfig config; + config.layerConfig.set_type("multibox_loss"); + config.biasSize = 0; + LayerInputConfig* input = config.layerConfig.add_inputs(); + MultiBoxLossConfig* multiboxLoss = input->mutable_multibox_loss_conf(); + multiboxLoss->set_num_classes(21); + multiboxLoss->set_input_num(1); + multiboxLoss->set_overlap_threshold(0.5); + multiboxLoss->set_neg_pos_ratio(3); + multiboxLoss->set_neg_overlap(0.5); + multiboxLoss->set_background_id(0); + multiboxLoss->set_height(3); + multiboxLoss->set_width(3); + + size_t gtNum = 1; + MatrixPtr labelValue = Matrix::create(gtNum, 6, false, false); + labelValue->randomizeUniform(); + labelValue->add(-0.5); + labelValue->sigmoid(*labelValue); + real* labelData = labelValue->getData(); + size_t labelWidth = labelValue->getWidth(); + for (size_t i = 0; i < gtNum; ++i) { + *(labelData + i * labelWidth) = std::rand() % 20 + 1; + *(labelData + i * labelWidth + 1) = 0.400259; + *(labelData + i * labelWidth + 2) = 0.377857; + *(labelData + i * labelWidth + 3) = 0.525712; + *(labelData + i * labelWidth + 4) = 0.519368; + } + vector seqStartPositions(gtNum + 1, 0); + for (size_t i = 1; i <= gtNum; ++i) { + seqStartPositions[i] = i; + } + + // Ensure at lease one matched bbox + MatrixPtr priorValue = Matrix::create(1, 72, false, false); + priorValue->randomizeUniform(); + priorValue->add(-0.5); + priorValue->sigmoid(*priorValue); + real* priorData = priorValue->getData(); + *(priorData) = 0.424811; + *(priorData + 1) = 0.397059; + *(priorData + 2) = 0.538905; + *(priorData + 3) = 0.447091; + *(priorData + 4) = 0.425720; + *(priorData + 5) = 0.515228; + *(priorData + 6) = 0.519452; + *(priorData + 7) = 0.591065; + + config.inputDefs.push_back( + {INPUT_SELF_DEFINE_DATA, "priorbox", priorValue, {}}); + config.inputDefs.push_back( + {INPUT_SELF_DEFINE_DATA, "label", labelValue, seqStartPositions}); + config.inputDefs.push_back({INPUT_DATA, "locPred", 36, 0}); + config.inputDefs.push_back({INPUT_DATA, "confPred", 189, 0}); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multibox_loss", 1, false, useGpu, false); + } +} + TEST(Layer, TransLayer) { TestConfig config; const int height = 128; diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 5beced3bb5a1050078f88dfd4350a2df71d27f35..ef72b973c1a465a8ac03cae1070429160eac0ac1 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -561,7 +561,7 @@ void Argument::degradeSequence(const Argument& input) { void Argument::poolSequenceWithStride(const Argument& input, size_t stride, - IVectorPtr* stridePostions, + ICpuGpuVectorPtr* stridePostions, bool reversed) { // If input.sequenceStartPositions = [0, 9, 14, 17, 30] and stride = 5, // then sequenceStartPositions = [0, 2, 3, 4, 7]. @@ -598,8 +598,8 @@ void Argument::poolSequenceWithStride(const Argument& input, stridePos.emplace_back(starts[numSequences]); int size = stridePos.size(); CHECK_EQ(size - 1, tgtBuf[numSequences]); - IVector::resizeOrCreate(*stridePostions, size, false); - (*stridePostions)->copyFrom(stridePos.data(), size); + ICpuGpuVector::resizeOrCreate(*stridePostions, size, false); + (*stridePostions)->getMutableVector(false)->copyFrom(stridePos.data(), size); } void Argument::getValueString( diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 09bd633616730dc9475edc596128166f4f70b0cd..0ccdef802e71b659788cfd24f28ebe43e1917db1 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -299,7 +299,7 @@ struct Argument { */ void poolSequenceWithStride(const Argument& input, size_t stride, - IVectorPtr* stridePositions, + ICpuGpuVectorPtr* stridePositions, bool reversed = false); /** * @brief getValueString will return the argument's output in string. There diff --git a/paddle/parameter/tests/test_argument.cpp b/paddle/parameter/tests/test_argument.cpp index 98ab013548734059060eb06ce1a7cec23dbf1b72..19df6ea95745609a4eb7487d422e61d2f0b269cc 100644 --- a/paddle/parameter/tests/test_argument.cpp +++ b/paddle/parameter/tests/test_argument.cpp @@ -31,7 +31,7 @@ TEST(Argument, poolSequenceWithStride) { int strideResultReversed[] = {0, 4, 9, 14, 17, 20, 25, 30}; for (auto reversed : {false, true}) { - IVectorPtr stridePositions; + ICpuGpuVectorPtr stridePositions; output.poolSequenceWithStride( input, 5 /* stride */, &stridePositions, reversed); @@ -45,7 +45,7 @@ TEST(Argument, poolSequenceWithStride) { CHECK_EQ(stridePositions->getSize(), 8UL); auto result = reversed ? strideResultReversed : strideResult; for (int i = 0; i < 8; i++) { - CHECK_EQ(stridePositions->getData()[i], result[i]); + CHECK_EQ(stridePositions->getData(false)[i], result[i]); } } } diff --git a/paddle/parameter/tests/test_common.cpp b/paddle/parameter/tests/test_common.cpp index 8bab5a6289e2bb9f634e8cce4557de55f7704447..64d204aea10c8a7905d90fac6ebccde3c9da1edc 100644 --- a/paddle/parameter/tests/test_common.cpp +++ b/paddle/parameter/tests/test_common.cpp @@ -172,53 +172,3 @@ TEST_F(CommonTest, syncThreadPool) { EXPECT_EQ((int)0, nums[i]); } } - -TEST_F(CommonTest, barrierStat) { - const int threadNum = 10; - - SyncThreadPool pool(threadNum); - -#define TEST_BARRIER_RANDOM(statName, numConnThreads, ...) \ - pool.exec([&](int tid, size_t numThreads) { \ - struct timeval time; \ - gettimeofday(&time, nullptr); \ - uint64_t usec = timeToMicroSecond(time); \ - std::srand(usec); \ - auto value = std::rand() % 100000; \ - usleep(value); \ - REGISTER_SLOW_NODES_PROBE( \ - globalStat, statName, numConnThreads, tid, __VA_ARGS__); \ - }); - - for (auto i = 0; i < 10; i++) { - TEST_BARRIER_RANDOM("synThreadBarrier1", threadNum); - TEST_BARRIER_RANDOM("synThreadBarrier2", threadNum); - } - - globalStat.printAllStatus(); - globalStat.reset(); - - for (auto i = 0; i < 10; i++) { - TEST_BARRIER_RANDOM("synThreadBarrier3", threadNum, "tag0"); - TEST_BARRIER_RANDOM("synThreadBarrier4", threadNum, "tag1"); - } - - globalStat.printAllStatus(); - globalStat.reset(); - -// use it to test accurate barrier gap -#define TEST_BARRIER(statName, numConnThreads, ...) \ - pool.exec([&](int tid, size_t numThreads) { \ - usleep(tid * 10000); \ - REGISTER_SLOW_NODES_PROBE( \ - globalStat, statName, numConnThreads, tid, __VA_ARGS__); \ - }); - - for (auto i = 0; i < 10; i++) { - TEST_BARRIER("synThreadBarrier3", threadNum, "tag0"); - TEST_BARRIER("synThreadBarrier4", threadNum, "tag1"); - } - - globalStat.printAllStatus(); - globalStat.reset(); -} diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt index c7d7b14518ebb8415014a78fc1a3bafa8c386191..cc6b52e9271ff00e91f2ca172815c543eb99261d 100644 --- a/paddle/platform/CMakeLists.txt +++ b/paddle/platform/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(dynload) + nv_test(cuda_test SRCS cuda_test.cu) cc_library(place SRCS place.cc) diff --git a/paddle/platform/cuda.h b/paddle/platform/cuda.h index 8fe891f9ce6c3add1df48a8b1f79fd811c7a4362..5ed36c0f02549e29fc680eba097c9d851fa346e5 100644 --- a/paddle/platform/cuda.h +++ b/paddle/platform/cuda.h @@ -34,6 +34,16 @@ int GetDeviceCount(void) { return count; } +int GetCurrentDeviceId(void) { + int device_id; + throw_on_error(cudaGetDevice(&device_id), "cudaGetDevice failed"); + return device_id; +} + +void SetDeviceId(int device_id) { + throw_on_error(cudaSetDevice(device_id), "cudaSetDevice failed"); +} + } // namespace platform } // namespace paddle diff --git a/paddle/platform/dynload/CMakeLists.txt b/paddle/platform/dynload/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9f829b70128655f018c59db32b95d3f1789da5fc --- /dev/null +++ b/paddle/platform/dynload/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags) diff --git a/paddle/platform/dynload/cublas.h b/paddle/platform/dynload/cublas.h new file mode 100644 index 0000000000000000000000000000000000000000..258cc88031a71e9fee65b5445bd5537d6782e226 --- /dev/null +++ b/paddle/platform/dynload/cublas.h @@ -0,0 +1,106 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include "paddle/platform/dynload/dynamic_loader.h" + +namespace paddle { +namespace platform { +namespace dynload { + +std::once_flag cublas_dso_flag; +void *cublas_dso_handle = nullptr; + +/** + * The following macro definition can generate structs + * (for each function) to dynamic load cublas routine + * via operator overloading. + * + * note: default dynamic linked libs + */ +#ifdef PADDLE_USE_DSO +#define DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + cublasStatus_t operator()(Args... args) { \ + typedef cublasStatus_t (*cublasFunc)(Args...); \ + std::call_once(cublas_dso_flag, \ + paddle::platform::dynload::GetCublasDsoHandle, \ + &cublas_dso_handle); \ + void *p_##__name = dlsym(cublas_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + } __name; // struct DynLoad__##__name +#else +#define DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + cublasStatus_t operator()(Args... args) { \ + return __name(args...); \ + } \ + } __name; // struct DynLoad__##__name +#endif + +#define DYNAMIC_LOAD_CUBLAS_V2_WRAP(__name) DYNAMIC_LOAD_CUBLAS_WRAP(__name) + +// include all needed cublas functions in HPPL +// clang-format off +#define CUBLAS_BLAS_ROUTINE_EACH(__macro) \ + __macro(cublasSgemv) \ + __macro(cublasDgemv) \ + __macro(cublasSgemm) \ + __macro(cublasDgemm) \ + __macro(cublasSgeam) \ + __macro(cublasDgeam) \ + +DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasCreate) +DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasDestroy) +DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasSetStream) +DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasSetPointerMode) +DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasGetPointerMode) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasCgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasZgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetriBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetriBatched) +CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) + +#undef DYNAMIC_LOAD_CUBLAS_WRAP +#undef DYNAMIC_LOAD_CUBLAS_V2_WRAP +#undef CUBLAS_BLAS_ROUTINE_EACH + +// clang-format on +#ifndef PADDLE_TYPE_DOUBLE +#define CUBLAS_GEAM paddle::platform::dynload::cublasSgeam +#define CUBLAS_GEMV paddle::platform::dynload::cublasSgemv +#define CUBLAS_GEMM paddle::platform::dynload::cublasSgemm +#define CUBLAS_GETRF paddle::platform::dynload::cublasSgetrfBatched +#define CUBLAS_GETRI paddle::platform::dynload::cublasSgetriBatched +#else +#define CUBLAS_GEAM paddle::platform::dynload::cublasDgeam +#define CUBLAS_GEMV paddle::platform::dynload::cublasDgemv +#define CUBLAS_GEMM paddle::platform::dynload::cublasDgemm +#define CUBLAS_GETRF paddle::platform::dynload::cublasDgetrfBatched +#define CUBLAS_GETRI paddle::platform::dynload::cublasDgetriBatched +#endif +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/cudnn.h b/paddle/platform/dynload/cudnn.h new file mode 100644 index 0000000000000000000000000000000000000000..0a9562c573cdfe059ef7caa39ba62efa87225e41 --- /dev/null +++ b/paddle/platform/dynload/cudnn.h @@ -0,0 +1,136 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include "paddle/platform/dynload/dynamic_loader.h" + +namespace paddle { +namespace platform { +namespace dynload { + +std::once_flag cudnn_dso_flag; +void* cudnn_dso_handle = nullptr; + +#ifdef PADDLE_USE_DSO + +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudnn_func = decltype(__name(args...)) (*)(Args...); \ + std::call_once(cudnn_dso_flag, \ + paddle::platform::dynload::GetCudnnDsoHandle, \ + &cudnn_dso_handle); \ + void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + } __name; /* struct DynLoad__##__name */ + +#else + +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + return __name(args...); \ + } \ + } __name; /* struct DynLoad__##__name */ + +#endif + +/** + * include all needed cudnn functions in HPPL + * different cudnn version has different interfaces + **/ +// clang-format off +#define CUDNN_DNN_ROUTINE_EACH(__macro) \ + __macro(cudnnSetTensor4dDescriptor) \ + __macro(cudnnSetTensor4dDescriptorEx) \ + __macro(cudnnGetConvolutionNdForwardOutputDim) \ + __macro(cudnnGetConvolutionForwardAlgorithm) \ + __macro(cudnnCreateTensorDescriptor) \ + __macro(cudnnDestroyTensorDescriptor) \ + __macro(cudnnCreateFilterDescriptor) \ + __macro(cudnnSetFilter4dDescriptor) \ + __macro(cudnnSetPooling2dDescriptor) \ + __macro(cudnnDestroyFilterDescriptor) \ + __macro(cudnnCreateConvolutionDescriptor) \ + __macro(cudnnCreatePoolingDescriptor) \ + __macro(cudnnDestroyPoolingDescriptor) \ + __macro(cudnnSetConvolution2dDescriptor) \ + __macro(cudnnDestroyConvolutionDescriptor) \ + __macro(cudnnCreate) \ + __macro(cudnnDestroy) \ + __macro(cudnnSetStream) \ + __macro(cudnnActivationForward) \ + __macro(cudnnConvolutionForward) \ + __macro(cudnnConvolutionBackwardBias) \ + __macro(cudnnGetConvolutionForwardWorkspaceSize) \ + __macro(cudnnTransformTensor) \ + __macro(cudnnPoolingForward) \ + __macro(cudnnPoolingBackward) \ + __macro(cudnnSoftmaxBackward) \ + __macro(cudnnSoftmaxForward) \ + __macro(cudnnGetVersion) \ + __macro(cudnnGetErrorString) +CUDNN_DNN_ROUTINE_EACH(DYNAMIC_LOAD_CUDNN_WRAP) + +#define CUDNN_DNN_ROUTINE_EACH_R2(__macro) \ + __macro(cudnnAddTensor) \ + __macro(cudnnConvolutionBackwardData) \ + __macro(cudnnConvolutionBackwardFilter) +CUDNN_DNN_ROUTINE_EACH_R2(DYNAMIC_LOAD_CUDNN_WRAP) + +// APIs available after R3: +#if CUDNN_VERSION >= 3000 +#define CUDNN_DNN_ROUTINE_EACH_AFTER_R3(__macro) \ + __macro(cudnnGetConvolutionBackwardFilterWorkspaceSize) \ + __macro(cudnnGetConvolutionBackwardDataAlgorithm) \ + __macro(cudnnGetConvolutionBackwardFilterAlgorithm) \ + __macro(cudnnGetConvolutionBackwardDataWorkspaceSize) +CUDNN_DNN_ROUTINE_EACH_AFTER_R3(DYNAMIC_LOAD_CUDNN_WRAP) +#undef CUDNN_DNN_ROUTINE_EACH_AFTER_R3 +#endif + + +// APIs available after R4: +#if CUDNN_VERSION >= 4007 +#define CUDNN_DNN_ROUTINE_EACH_AFTER_R4(__macro) \ + __macro(cudnnBatchNormalizationForwardTraining) \ + __macro(cudnnBatchNormalizationForwardInference) \ + __macro(cudnnBatchNormalizationBackward) +CUDNN_DNN_ROUTINE_EACH_AFTER_R4(DYNAMIC_LOAD_CUDNN_WRAP) +#undef CUDNN_DNN_ROUTINE_EACH_AFTER_R4 +#endif + +// APIs in R5 +#if CUDNN_VERSION >= 5000 +#define CUDNN_DNN_ROUTINE_EACH_R5(__macro) \ + __macro(cudnnCreateActivationDescriptor) \ + __macro(cudnnSetActivationDescriptor) \ + __macro(cudnnGetActivationDescriptor) \ + __macro(cudnnDestroyActivationDescriptor) +CUDNN_DNN_ROUTINE_EACH_R5(DYNAMIC_LOAD_CUDNN_WRAP) +#undef CUDNN_DNN_ROUTINE_EACH_R5 +#endif + +#undef CUDNN_DNN_ROUTINE_EACH +// clang-format on +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/curand.h b/paddle/platform/dynload/curand.h new file mode 100644 index 0000000000000000000000000000000000000000..9dc0a25c0fbdc3f73f1dd82206e8940972b0b7f5 --- /dev/null +++ b/paddle/platform/dynload/curand.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include "paddle/platform/dynload/dynamic_loader.h" + +namespace paddle { +namespace platform { +namespace dynload { +std::once_flag curand_dso_flag; +void *curand_dso_handle = nullptr; +#ifdef PADDLE_USE_DSO +#define DYNAMIC_LOAD_CURAND_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + curandStatus_t operator()(Args... args) { \ + typedef curandStatus_t (*curandFunc)(Args...); \ + std::call_once(curand_dso_flag, \ + paddle::platform::dynload::GetCurandDsoHandle, \ + &curand_dso_handle); \ + void *p_##__name = dlsym(curand_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + } __name; /* struct DynLoad__##__name */ +#else +#define DYNAMIC_LOAD_CURAND_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + curandStatus_t operator()(Args... args) { \ + return __name(args...); \ + } \ + } __name; /* struct DynLoad__##__name */ +#endif + +/* include all needed curand functions in HPPL */ +// clang-format off +#define CURAND_RAND_ROUTINE_EACH(__macro) \ + __macro(curandCreateGenerator) \ + __macro(curandSetStream) \ + __macro(curandSetPseudoRandomGeneratorSeed)\ + __macro(curandGenerateUniform) \ + __macro(curandGenerateUniformDouble) \ + __macro(curandDestroyGenerator) +// clang-format on + +CURAND_RAND_ROUTINE_EACH(DYNAMIC_LOAD_CURAND_WRAP) + +#undef CURAND_RAND_ROUTINE_EACH +#undef DYNAMIC_LOAD_CURAND_WRAP +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/dynamic_loader.cc b/paddle/platform/dynload/dynamic_loader.cc new file mode 100644 index 0000000000000000000000000000000000000000..dd914e006d54c423ffea56ffaaafe7dcba416361 --- /dev/null +++ b/paddle/platform/dynload/dynamic_loader.cc @@ -0,0 +1,162 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/platform/dynload/dynamic_loader.h" +#include +#include +#include +#include +#include "gflags/gflags.h" +#include "glog/logging.h" +#include "paddle/framework/enforce.h" + +DEFINE_string(cudnn_dir, "", + "Specify path for loading libcudnn.so. For instance, " + "/usr/local/cudnn/lib. If empty [default], dlopen " + "will search cudnn from LD_LIBRARY_PATH"); + +DEFINE_string(cuda_dir, "", + "Specify path for loading cuda library, such as libcublas, " + "libcurand. For instance, /usr/local/cuda/lib64. If default, " + "dlopen will search cuda from LD_LIBRARY_PATH"); + +DEFINE_string(warpctc_dir, "", "Specify path for loading libwarpctc.so."); + +DEFINE_string(lapack_dir, "", "Specify path for loading liblapack.so."); + +namespace paddle { +namespace platform { +namespace dynload { + +static inline std::string join(const std::string& part1, + const std::string& part2) { + // directory separator + const char sep = '/'; + if (!part2.empty() && part2.front() == sep) { + return part2; + } + std::string ret; + ret.reserve(part1.size() + part2.size() + 1); + ret = part1; + if (!ret.empty() && ret.back() != sep) { + ret += sep; + } + ret += part2; + return ret; +} + +static inline void GetDsoHandleFromDefaultPath(std::string& dso_path, + void** dso_handle, + int dynload_flags) { + VLOG(3) << "Try to find library: " << dso_path + << " from default system path."; + // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + +// DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to +// bring System Integrity Projection (SIP), if dso_handle +// is null, search from default package path in Mac OS. +#if defined(__APPLE__) || defined(__OSX__) + if (nullptr == *dso_handle) { + dso_path = join("/usr/local/cuda/lib/", dso_path); + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + if (nullptr == *dso_handle) { + if (dso_path == "libcudnn.dylib") { + PADDLE_ENFORCE(true, + "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n " + "For instance, sudo tar -xzf " + "cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local \n sudo " + "chmod a+r /usr/local/cuda/include/cudnn.h " + "/usr/local/cuda/lib/libcudnn*"); + } + } + } +#endif +} + +static inline void GetDsoHandleFromSearchPath(const std::string& search_root, + const std::string& dso_name, + void** dso_handle) { + int dynload_flags = RTLD_LAZY | RTLD_LOCAL; + *dso_handle = nullptr; + + std::string dlPath = dso_name; + if (search_root.empty()) { + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); + } else { + // search xxx.so from custom path + dlPath = join(search_root, dso_name); + *dso_handle = dlopen(dlPath.c_str(), dynload_flags); + // if not found, search from default path + if (nullptr == *dso_handle) { + LOG(WARNING) << "Failed to find dynamic library: " << dlPath << " (" + << dlerror() << ")"; + dlPath = dso_name; + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); + } + } + PADDLE_ENFORCE(nullptr != *dso_handle, + "Failed to find dynamic library: %s ( %s ) \n Please specify " + "its path correctly using following ways: \n Method. set " + "environment variable LD_LIBRARY_PATH on Linux or " + "DYLD_LIBRARY_PATH on Mac OS. \n For instance, issue command: " + "export LD_LIBRARY_PATH=... \n Note: After Mac OS 10.11, " + "using the DYLD_LIBRARY_PATH is impossible unless System " + "Integrity Protection (SIP) is disabled.", + dlPath, dlerror()); +} + +void GetCublasDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); +#else + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); +#endif +} + +void GetCudnnDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); +#else + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); +#endif +} + +void GetCurandDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); +#else + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); +#endif +} + +void GetWarpCTCDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.dylib", dso_handle); +#else + GetDsoHandleFromSearchPath(FLAGS_warpctc_dir, "libwarpctc.so", dso_handle); +#endif +} + +void GetLapackDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.dylib", dso_handle); +#else + GetDsoHandleFromSearchPath(FLAGS_lapack_dir, "liblapacke.so", dso_handle); +#endif +} + +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/dynamic_loader.h b/paddle/platform/dynload/dynamic_loader.h new file mode 100644 index 0000000000000000000000000000000000000000..a99b05443feb909f10b2c56f4d8bdf3c6fa11e3f --- /dev/null +++ b/paddle/platform/dynload/dynamic_loader.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +namespace paddle { +namespace platform { +namespace dynload { + +/** + * @brief load the DSO of CUBLAS + * + * @param **dso_handle dso handler + * + */ +void GetCublasDsoHandle(void** dso_handle); + +/** + * @brief load the DSO of CUDNN + * + * @param **dso_handle dso handler + * + */ +void GetCudnnDsoHandle(void** dso_handle); + +/** + * @brief load the DSO of CURAND + * + * @param **dso_handle dso handler + * + */ +void GetCurandDsoHandle(void** dso_handle); + +/** + * @brief load the DSO of warp-ctc + * + * @param **dso_handle dso handler + * + */ +void GetWarpCTCDsoHandle(void** dso_handle); + +/** + * @brief load the DSO of lapack + * + * @param **dso_handle dso handler + * + */ +void GetLapackDsoHandle(void** dso_handle); + +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index 922f25734dee0a6db7fbcfcef3d29d2bad5b7858..8616fd2d5aef666f16533fe062f3f40a7a2b202d 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -142,7 +142,7 @@ SocketServer::SocketServer(const std::string &addr, int port, int rdmaCpu) } /// trigger to initialize RDMA lib - PCHECK(RdmaClientDaemons::get()) << "initilizate RDMA failed\n"; + CHECK(RdmaClientDaemons::get()) << "initilizate RDMA failed\n"; } SocketServer::~SocketServer() { @@ -168,7 +168,7 @@ void SocketServer::tcpServer() { /// First call to socket() function socket_ = socket(AF_INET, SOCK_STREAM, 0); - PCHECK(socket_ >= 0) << "ERROR opening socket"; + CHECK(socket_ >= 0) << "ERROR opening socket"; /// Initialize socket structure bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -176,7 +176,7 @@ void SocketServer::tcpServer() { serv_addr.sin_port = htons(port_); if (!addr_.empty()) { server = gethostbyname(addr_.c_str()); - PCHECK(server) << "ERROR, no such host: " << addr_; + CHECK(server) << "ERROR, no such host: " << addr_; bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); @@ -187,7 +187,7 @@ void SocketServer::tcpServer() { setOption(socket_); /// Now bind the host address using bind() call. - PCHECK(bind(socket_, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) >= 0) + CHECK(bind(socket_, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) >= 0) << "ERROR on binding " << addr_; /// Now start listening for the clients, here process will @@ -201,7 +201,7 @@ void SocketServer::tcpServer() { if (stopping_) { break; } - PCHECK(newsockfd >= 0) << "ERROR on accept"; + CHECK(newsockfd >= 0) << "ERROR on accept"; constexpr int kPeerNameLen = 128; char peerName[kPeerNameLen]; CHECK(inet_ntop(AF_INET, &cli_addr.sin_addr, peerName, kPeerNameLen)); @@ -227,14 +227,14 @@ void SocketServer::rdmaServer() { /// First call to socket() function rdmaSocket_ = rdma::ssocket(rdmaCpu_); - PCHECK(rdmaSocket_) << "ERROR opening RDMA socket"; + CHECK(rdmaSocket_) << "ERROR opening RDMA socket"; - PCHECK(rdma::bind(rdmaSocket_, rdmaUri_.c_str()) == 0) + CHECK(rdma::bind(rdmaSocket_, rdmaUri_.c_str()) == 0) << "ERROR bind RDMA socket"; /// Now start listening for the clients, here process will /// go in sleep mode and will wait for the incoming connection - PCHECK(rdma::listen(rdmaSocket_) == 0) << "ERROR listen RDMA socket"; + CHECK(rdma::listen(rdmaSocket_) == 0) << "ERROR listen RDMA socket"; while (true) { /// Accept actual connection from the client @@ -242,7 +242,7 @@ void SocketServer::rdmaServer() { if (stopping_) { break; } - PCHECK(newsock) << "ERROR on accept"; + CHECK(newsock) << "ERROR on accept"; constexpr int kPeerNameLen = 128; char peerName[kPeerNameLen]; @@ -290,7 +290,7 @@ RdmaClientDaemons::RdmaClientDaemons() { onlineCpus_ = rdma::numCpus(); for (auto i = 0; i < onlineCpus_; i++) { socket = rdma::csocket(i); - PCHECK(socket) << "ERROR open client socket daemon"; + CHECK(socket) << "ERROR open client socket daemon"; rdmaClientSocket_.push_back(socket); } @@ -355,7 +355,7 @@ void SocketClient::TcpClient(const std::string &serverAddr, int serverPort) { /// Create a socket point int sockfd = socket(AF_INET, SOCK_STREAM, 0); - PCHECK(sockfd >= 0) << "ERROR opening socket"; + CHECK(sockfd >= 0) << "ERROR opening socket"; #if defined(__OSX__) || defined(__APPLE__) server = getipnodebyname(serverAddr.c_str(), AF_INET, AI_DEFAULT, &errRet); @@ -396,8 +396,8 @@ void SocketClient::TcpClient(const std::string &serverAddr, int serverPort) { } std::this_thread::sleep_for(std::chrono::seconds(1)); } else { - PCHECK(errno != 0) << "ERROR connecting to " << serverAddr << ":" - << serverPort << "errorno: " << errno; + CHECK(errno != 0) << "ERROR connecting to " << serverAddr << ":" + << serverPort << "errorno: " << errno; } } while (errno == ECONNREFUSED); @@ -426,7 +426,7 @@ void SocketClient::RdmaClient(const std::string &serverAddr, int serverPort) { /// connect to server with socket daemon sock = rdma::connect(socketDaemon_, rdmaUri.c_str()); - PCHECK(sock) << "ERROR connect to server" << rdmaUri; + CHECK(sock) << "ERROR connect to server" << rdmaUri; std::vector seg; str::split(rdmaUri, '/', &seg); diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index 41ac15336d3150417da1cf1631319604584991ec..d7c1d4f788f44c6bfcec040ba24bdc454348c911 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -217,10 +217,6 @@ void ParameterServer2::setConfig(const SetConfigRequest& request, SetConfigResponse response; callback(response); - - /// always defined, barrier slowest node function need it. - statSet_.reset(new StatSet("ParameterServer" + - str::to_string(static_cast(serverId_)))); } real bufferSum(const std::vector& buffers) { @@ -369,50 +365,7 @@ void ParameterServer2::addGradient(const SendParameterRequest& request, std::vector* outputBuffers) { VLOG(1) << "pserver: addGradient"; - // forwardbackward delta from all trainers - // indicate the fluctuation caused by forwardbackward. - if (!numPassFinishClients_) { - REGISTER_BARRIER_DELTA_SERVER_SET( - *statSet_, - "forwardbackwardDelta", - FLAGS_num_gradient_servers, - request.trainer_id(), - request.forwardbackward_time(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - { - /// approximately pure network overhead - REGISTER_TIMER_DYNAMIC_SET( - "pushRecv", timeToMicroSecond(*handleRequestBegin_), -1, *statSet_); - } - -#ifndef PADDLE_DISABLE_TIMER - gettimeofday(&(*addGradBegin_), nullptr); -#endif - - /// barrier fluctuation caused by network and previous forwardbackward - if (!numPassFinishClients_) { - REGISTER_BARRIER_TIMER_SERVER_SET( - *statSet_, - "handleReqBegin", - FLAGS_num_gradient_servers, - request.trainer_id(), - (*handleRequestBegin_), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - - if (!numPassFinishClients_) { - REGISTER_BARRIER_TIMER_SERVER( - *statSet_, - "addGradBegin", - FLAGS_num_gradient_servers, - request.trainer_id(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - - { - REGISTER_TIMER_DYNAMIC("addGradCore", -1, *statSet_); ReadLockGuard guard(parameterMutex_); int bufferIndex = 0; for (const auto& block : request.blocks()) { @@ -444,15 +397,6 @@ void ParameterServer2::addGradient(const SendParameterRequest& request, std::lock_guard guard(*info.lock); simd::addTo(gradientSumBuffer, gradientBuffer, size); } - - if (!numPassFinishClients_) { - REGISTER_BARRIER_TIMER_SERVER( - *statSet_, - "addGradCoreFinish", - FLAGS_num_gradient_servers, - request.trainer_id(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } } if (request.batch_status() == BATCH_FINISH || request.batch_status() == BATCH_START_AND_FINISH) { @@ -461,47 +405,12 @@ void ParameterServer2::addGradient(const SendParameterRequest& request, VLOG(1) << "num samples: " << numSamplesProcessed_ << ", new cost:" << cost_; - /// numPassFinishClients_ means some trainer has entered finishPass - if (!numPassFinishClients_) { - REGISTER_SLOW_NODES_PROBE( - *statSet_, - "SLOW_NODES", - FLAGS_num_gradient_servers, - request.trainer_id(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - /// notify doOperation gradient ready gradientReadyBarrier_.wait(); - /// if wait pass finish does not start, do check - if (!numPassFinishClients_) { - CHECK_BARRIER_TIMER(*statSet_, - "SLOW_NODES", - FLAGS_num_gradient_servers, - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - - /// barrier performance while all parameter add is finished - /// can indicate the fluctation caused by computation at pserver. - if (!numPassFinishClients_) { - REGISTER_BARRIER_TIMER_SERVER( - *statSet_, - "paraReady", - FLAGS_num_gradient_servers, - request.trainer_id(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } /// wait doOperation finish parameterReadyBarrier_.wait(); VLOG(1) << "start send back"; - { - /// total time except overhead of network. - REGISTER_TIMER_DYNAMIC_SET("sendParaNoRecvNoSend", - timeToMicroSecond(*addGradBegin_), - -1, - *statSet_); - } } } @@ -543,57 +452,6 @@ bool ParameterServer2::asyncGrdientCommitCheckAndStat( return commitGradient; } -void ParameterServer2::printAsyncGradientCommitStatAndReset() { - std::stringstream statFormat; - if (asyncUpdateSteps_) { - statFormat << "async discard gradients stat: " << std::endl; - statFormat << "serverId: " << serverId_ - << " serverType: " << isSparseServer_ - << " total updates: " << asyncUpdateSteps_ - << " discard updates: " << asyncLaggedGradientsNum_ - << " discard ratio: " - << (real)asyncLaggedGradientsNum_ / (real)asyncUpdateSteps_; - statFormat << std::endl; - statFormat << std::endl; - - statFormat << "Async Gradient Update Steps distribution: " << std::endl - << "Sample: 1:1912(0.00284449) means " - << "the updates step=1 count 1912 times " - << "and account for 0.284449% of total updates" << std::endl; - size_t index = 0; - for (const auto& stat : asyncUpdateStat_) { - statFormat << index << ":" << stat << "(" - << (real)stat / (real)asyncUpdateSteps_ << ") "; - index++; - } - statFormat << std::endl; - statFormat << std::endl; - - statFormat << "Async Gradient Discard based on trainer_id: " << std::endl - << "Sample: 2:22(0.0016363) means " - << "total discarded updates from trainer_id=2 count 22 " - << "and account for 0.16363% of all updates from trainer_id=2" - << std::endl; - for (auto i = 0; i < FLAGS_num_gradient_servers; i++) { - real ratio = - (real)asyncTrainerDiscardStat_[i] / - (real)(asyncTrainerCommitStat_[i] + asyncTrainerDiscardStat_[i]); - statFormat << i << ":" << asyncTrainerDiscardStat_[i] << "(" << ratio - << ")" - << " "; - } - LOG(INFO) << statFormat.str(); - - /// reset stat - asyncUpdateSteps_ = 0; - asyncTrainerSteps_.assign(asyncTrainerSteps_.size(), 0); - asyncLaggedGradientsNum_ = 0; - asyncUpdateStat_.assign(asyncUpdateStat_.size(), 0); - asyncTrainerDiscardStat_.assign(asyncTrainerDiscardStat_.size(), 0); - asyncTrainerCommitStat_.assign(asyncTrainerCommitStat_.size(), 0); - } -} - static ThreadLocal> localBlockBitset_; void ParameterServer2::asyncSGD(const SendParameterRequest& request, @@ -695,7 +553,6 @@ void ParameterServer2::asyncSGD(const SendParameterRequest& request, if (request.trainer_id() == 0) { /// batchId_ is approximately equal to "real batchId_" batchId_++; - tuningAsyncsgdMidOutput(); } } @@ -881,34 +738,6 @@ void ParameterServer2::sendParameter(const SendParameterRequest& request, } (*requestVec_).clear(); (*callbackVec_).clear(); - - /// barrier perfromance while all data are send finished. - /// indicates network flucatuation for big message. - if (!numPassFinishClients_) { - REGISTER_BARRIER_TIMER_SERVER( - *statSet_, - "sendParamFinish", - FLAGS_num_gradient_servers, - request.trainer_id(), - isSparseServer_ ? "_sparseUpdater" : "_denseUpdater"); - } - /// all time exhausted in parameterServer for big message. - /// it contains network and computation at pserver. - { - /// total time including overhead of network. - REGISTER_TIMER_DYNAMIC_SET("sendParaTotal", - timeToMicroSecond(*handleRequestBegin_), - -1, - *statSet_); - } - /// all time exhausted in pserverServer except recieve network. - { - /// total time except overhead of network receive - REGISTER_TIMER_DYNAMIC_SET("sendParaNoRecv", - timeToMicroSecond(*addGradBegin_), - -1, - *statSet_); - } } break; case PSERVER_UPDATE_MODE_SET_PARAM: @@ -1088,8 +917,6 @@ void ParameterServer2::op_SGD(const Operation& operation, } { - REGISTER_TIMER_DYNAMIC("op_SGD", -1, *statSet_); - parallelExecForEachBlock([&](int64_t blockId, const VectorPtr vecs[]) { BlockInfo& info = blockInfos_[blockId]; const ParameterConfig& config = getParameterConfig(blockId); @@ -1113,7 +940,6 @@ void ParameterServer2::op_SGD(const Operation& operation, } batchId_++; - tuningSgdMidOutput(); } void ParameterServer2::op_start_pass(const Operation& operation, @@ -1146,8 +972,6 @@ void ParameterServer2::op_finish_pass(const Operation& operation, /// finish pass info.optimizer->finishPass(); }); - - tuningSgdFinished(); batchId_ = 0; } @@ -1515,7 +1339,6 @@ void ParameterServer2::asyncFinishPass(const SynchronizeRequest& request, callback(SynchronizeResponse()); if (request.trainer_id() == 0) { - tuningAsyncsgdFinished(); batchId_ = 0; } } @@ -1574,42 +1397,4 @@ void ParameterServer2::releaseMatrix(const ReleaseMatrixRequest& request, callback(response); } -void ParameterServer2::tuningSgdMidOutput() { - if (batchId_ && batchId_ % FLAGS_log_period_server == 0) { - LOG(INFO) << "======== Batch=" << batchId_ << "======="; - statSet_->setThreadInfo(true); - statSet_->printAllStatus(); - /// not reset raw data for reducing the overhead of performance tuning - statSet_->reset(false); - } -} - -void ParameterServer2::tuningSgdFinished() { - LOG(INFO) << "======== Batch=" << batchId_ << " pass END" - << "======="; - statSet_->setThreadInfo(true); - statSet_->printAllStatus(); - /** - * reset raw data at end of pass since some raw data could be not - * complete. Otherwise the raw data will pollute next pass performance - * tuning - */ - statSet_->reset(); -} - -void ParameterServer2::tuningAsyncsgdMidOutput() { -#ifndef PADDLE_DISABLE_TIMER - if (batchId_ && batchId_ % FLAGS_log_period_server == 0) { - LOG(INFO) << "======== [not accurate] Batch=" << batchId_ << "======="; - printAsyncGradientCommitStatAndReset(); - } -#endif -} - -void ParameterServer2::tuningAsyncsgdFinished() { - LOG(INFO) << "======== [not accurate] Batch=" << batchId_ << " pass END" - << "======="; - printAsyncGradientCommitStatAndReset(); -} - } // namespace paddle diff --git a/paddle/pserver/ParameterServer2.h b/paddle/pserver/ParameterServer2.h index 0f5a5895907b20a0cf882b6fa6fb74bd52dce058..f7d3587b88c4ab1d4e37a259c622fc7c2d5532a3 100644 --- a/paddle/pserver/ParameterServer2.h +++ b/paddle/pserver/ParameterServer2.h @@ -298,24 +298,6 @@ protected: /// barrier performance tuning sync-sgd required std::atomic batchId_; - /// the beginning of addGradient without network overhead - ThreadLocal addGradBegin_; - - /** - * tuning barrier performance - * to better control log for sparse and dense parameter, - * we use different log entities for different parameterServer - * objects. - * it will output lots of performance stats to perceive the - * overhead of network, fluctuation of computation from - * forwardbackward and network, computation from optimization - * at pserver end, barrier overhead, etc. to understand tuning - * data, focus on the synchronization between addGradient and - * doOperation which indirectly call op_SGD operation controlled - * by remote updater controller - */ - std::unique_ptr statSet_; - public: struct Buffer { real* base; @@ -325,7 +307,6 @@ public: protected: /// async gradient commit control bool asyncGrdientCommitCheckAndStat(const SendParameterRequest& request); - void printAsyncGradientCommitStatAndReset(); public: /// disable default parameter for overloading @@ -710,36 +691,6 @@ public: void op_load(const Operation& operation, OperationResult* result); void op_save(const Operation& operation, OperationResult* result); - - /** - * @brief output log in at the middle stage of training - * - * @note flush log histroy and state at the end for sgd - */ - void tuningSgdMidOutput(); - - /** - * @brief output log in at the end stage of training - * - * @note flush log histroy and state at the end for sgd. it will also - * flush some stateful stat for next pass. - */ - void tuningSgdFinished(); - - /** - * @brief output log in at the middle stage of training - * - * @note flush log histroy and state at the end for async-sgd. - * it will log some performance log if some lagged node are found - */ - void tuningAsyncsgdMidOutput(); - - /** - * @brief output log in at the end stage of training - * - * @note flush log histroy and state at the end for async-sgd. - */ - void tuningAsyncsgdFinished(); }; } // namespace paddle diff --git a/paddle/pserver/SocketChannel.cpp b/paddle/pserver/SocketChannel.cpp index 05998891649cee30e23e556d9311c3a383f43e10..12e3bc6552fcf26d8ccb32ca43d23142e3aba8e0 100644 --- a/paddle/pserver/SocketChannel.cpp +++ b/paddle/pserver/SocketChannel.cpp @@ -51,7 +51,7 @@ size_t SocketChannel::read(void* buf, size_t size) { else len = rdma::read(rdmaSocket_, (char*)buf + total, size - total); - PCHECK(len >= 0) << " peer=" << peerName_; + CHECK(len >= 0) << " peer=" << peerName_; if (len <= 0) { return total; } @@ -69,7 +69,7 @@ size_t SocketChannel::write(const void* buf, size_t size) { else len = rdma::write(rdmaSocket_, (char*)buf + total, size - total); - PCHECK(len >= 0) << " peer=" << peerName_; + CHECK(len >= 0) << " peer=" << peerName_; if (len <= 0) { return total; } @@ -98,10 +98,10 @@ static size_t readwritev(IOFunc iofunc, while (size < total) { ssize_t len = iofunc(socket, &iovs[curIov], std::min(iovcnt - curIov, maxiovs)); - PCHECK(len > 0) << " peer=" << peerName << " curIov=" << curIov - << " iovCnt=" << iovcnt - << " iovs[curIov].base=" << iovs[curIov].iov_base - << " iovs[curIov].iov_len=" << iovs[curIov].iov_len; + CHECK(len > 0) << " peer=" << peerName << " curIov=" << curIov + << " iovCnt=" << iovcnt + << " iovs[curIov].base=" << iovs[curIov].iov_base + << " iovs[curIov].iov_len=" << iovs[curIov].iov_len; size += len; /// restore iovs[curIov] to the original value @@ -183,7 +183,7 @@ void SocketChannel::writeMessage(const std::vector& userIovs) { header.totalLength += iov.iov_len; } - PCHECK(writev(iovs) == (size_t)header.totalLength); + CHECK(writev(iovs) == (size_t)header.totalLength); } std::unique_ptr SocketChannel::readMessage() { @@ -194,7 +194,7 @@ std::unique_ptr SocketChannel::readMessage() { return nullptr; } - PCHECK(len == sizeof(header)); + CHECK(len == sizeof(header)); std::unique_ptr msgReader(new MsgReader(this, header.numIovs)); @@ -209,7 +209,7 @@ std::unique_ptr SocketChannel::readMessage() { MsgReader::MsgReader(SocketChannel* channel, size_t numBlocks) : channel_(channel), blockLengths_(numBlocks), currentBlockIndex_(0) { size_t size = numBlocks * sizeof(blockLengths_[0]); - PCHECK(channel_->read(&blockLengths_[0], size) == size); + CHECK(channel_->read(&blockLengths_[0], size) == size); } void MsgReader::readBlocks(const std::vector& bufs) { @@ -223,12 +223,12 @@ void MsgReader::readBlocks(const std::vector& bufs) { ++currentBlockIndex_; } - PCHECK(channel_->readv(&iovs) == totalLength); + CHECK(channel_->readv(&iovs) == totalLength); } void MsgReader::readNextBlock(void* buf) { CHECK_LT(currentBlockIndex_, blockLengths_.size()); - PCHECK(channel_->read(buf, getNextBlockLength()) == getNextBlockLength()); + CHECK(channel_->read(buf, getNextBlockLength()) == getNextBlockLength()); ++currentBlockIndex_; } diff --git a/paddle/pserver/test/SocketTest.cpp b/paddle/pserver/test/SocketTest.cpp index 066a6c02939695e7050a7693365d7c449f70e723..6f6c9e596cfb7a2547d5b6c5de69381eb9c29132 100644 --- a/paddle/pserver/test/SocketTest.cpp +++ b/paddle/pserver/test/SocketTest.cpp @@ -113,7 +113,7 @@ void SocketServer::run() { /* First call to socket() function */ socket_ = socket(AF_INET, SOCK_STREAM, 0); - PCHECK(socket_ >= 0) << "ERROR opening socket"; + CHECK(socket_ >= 0) << "ERROR opening socket"; /* Initialize socket structure */ bzero((char*)&serv_addr, sizeof(serv_addr)); @@ -122,7 +122,7 @@ void SocketServer::run() { serv_addr.sin_port = htons(port_); /* Now bind the host address using bind() call.*/ - PCHECK(bind(socket_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) >= 0) + CHECK(bind(socket_, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) >= 0) << "ERROR on binding"; /* Now start listening for the clients, here process will @@ -134,7 +134,7 @@ void SocketServer::run() { while (true) { /* Accept actual connection from the client */ newsockfd = accept(socket_, (struct sockaddr*)&cli_addr, &clilen); - PCHECK(newsockfd >= 0) << "ERROR on accept"; + CHECK(newsockfd >= 0) << "ERROR on accept"; SocketWorker* worker = new SocketWorker(newsockfd); worker->start(); @@ -146,17 +146,17 @@ void SocketWorker::run() { while (true) { int64_t n = channel_.readAll(&header, sizeof(header)); - PCHECK(n == sizeof(header)) << "ERROR reading from socket"; + CHECK(n == sizeof(header)) << "ERROR reading from socket"; buffer_.resize(header.dataLength); n = channel_.readAll(&buffer_[0], header.dataLength); - PCHECK(n == header.dataLength) << "ERROR reading from socket"; + CHECK(n == header.dataLength) << "ERROR reading from socket"; /* Write a response to the client */ n = channel_.writeAll(&header, sizeof(header)); - PCHECK(n == sizeof(header)) << "ERROR reading from socket"; + CHECK(n == sizeof(header)) << "ERROR reading from socket"; n = channel_.writeAll(buffer_.data(), buffer_.size()); - PCHECK(n == header.dataLength) << "ERROR writing to socket"; + CHECK(n == header.dataLength) << "ERROR writing to socket"; } } @@ -177,9 +177,9 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) { /* Create a socket point */ int sockfd = socket(AF_INET, SOCK_STREAM, 0); - PCHECK(sockfd >= 0) << "ERROR opening socket"; + CHECK(sockfd >= 0) << "ERROR opening socket"; server = gethostbyname(serverAddr.c_str()); - PCHECK(server) << "ERROR, no such host: " << serverAddr; + CHECK(server) << "ERROR, no such host: " << serverAddr; bzero((char*)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; @@ -189,7 +189,7 @@ SocketClient::SocketClient(const std::string& serverAddr, int serverPort) { serv_addr.sin_port = htons(serverPort); /* Now connect to the server */ - PCHECK(connect(sockfd, (sockaddr*)&serv_addr, sizeof(serv_addr)) >= 0) + CHECK(connect(sockfd, (sockaddr*)&serv_addr, sizeof(serv_addr)) >= 0) << "ERROR connecting"; channel_.reset(new SocketChannel(sockfd)); @@ -234,18 +234,18 @@ int main(int argc, char** argv) { cpuGrad.copyFrom(gpuGrad); header.dataLength = dataSize; - PCHECK(channel->writeAll(&header, sizeof(header)) == sizeof(header)) + CHECK(channel->writeAll(&header, sizeof(header)) == sizeof(header)) << "Client write header error"; - PCHECK(channel->writeAll(cpuGrad.getData(), dataSize) == dataSize) + CHECK(channel->writeAll(cpuGrad.getData(), dataSize) == dataSize) << "Client write data error"; /* Now read server response */ - PCHECK(channel->readAll(&header, sizeof(header)) == sizeof(header)) + CHECK(channel->readAll(&header, sizeof(header)) == sizeof(header)) << "Client read header error"; CHECK_EQ((uint64_t)header.dataLength, dataSize); - PCHECK(channel->readAll(cpuParam.getData(), dataSize) == dataSize) + CHECK(channel->readAll(cpuParam.getData(), dataSize) == dataSize) << "Client read data error"; gpuParam.copyFrom(cpuParam); diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index a182e5f4aef9de8c6f20681328d5ba6c0e6944ef..ab60f1a38dd4cd1d9799c0019dccae5f1c7d4310 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -3,7 +3,7 @@ set -xe # Set BASE_IMAGE according to env variables -if [ ${WITH_GPU} == "ON" ]; then +if [[ ${WITH_GPU} == "ON" ]]; then BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04" else BASE_IMAGE="ubuntu:16.04" @@ -78,7 +78,7 @@ paddle version # PaddlePaddle. This awkwardness is due to # https://github.com/PaddlePaddle/Paddle/issues/1854. It also # describes a solution. -if [ ${WITH_DOC} == "ON" ]; then +if [[ ${WITH_DOC} == "ON" ]]; then cat <getHeight(); for (size_t i = 0; i < sampleNum; ++i) { diff --git a/paddle/utils/BarrierStat.cpp b/paddle/utils/BarrierStat.cpp deleted file mode 100644 index a6dbdcae3f32c894d35e8114488d4a3264c6c5f2..0000000000000000000000000000000000000000 --- a/paddle/utils/BarrierStat.cpp +++ /dev/null @@ -1,340 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/utils/BarrierStat.h" -#include -#include -#include -#include -#include "paddle/utils/Flags.h" -#include "paddle/utils/Stat.h" - -DEFINE_bool(log_barrier_abstract, - true, - "if true, show abstract of barrier performance"); -DEFINE_int32(log_barrier_lowest_nodes, - 5, - "how many lowest node will be logged"); -DEFINE_bool(log_barrier_show_log, - false, // for performance tuning insight - "if true, always show barrier abstract even with little gap"); - -namespace paddle { - -std::ostream &operator<<(std::ostream &output, const BarrierStatBase &stat) { - if (FLAGS_log_barrier_abstract) { - std::lock_guard guard(stat.lock_); - stat.showAbstract(output); - } - return output; -} - -BarrierStatBase::BarrierStatBase(uint16_t numConnThreads, - const std::string &name) - : totSamples_(0), numConnThreads_(numConnThreads), name_(name) { - abstract_.resize(numConnThreads_); - if (FLAGS_log_barrier_show_log) { - rateThreshold_ = 0.0; - } else { - /* probablity of abnormal node - * p = 1/n + (n/8)/(n+1), n = nodes, n > 1 - * if the freq of lowest trainerId larger than p, - * output FLAGS_log_barrier_lowest_nodes lastTrainerId. - * numConnThreads_ indicates nodes - */ - float n = (float)numConnThreads; - rateThreshold_ = 1.0 / n + (n / 8.0) / (n + 1.0); - } -} - -BarrierEndStat::BarrierEndStat(uint16_t numConnThreads, const std::string &name) - : BarrierStatBase(numConnThreads, name) { - timeVector_.reset(new TimeVectorEnd(numConnThreads_)); - reset(true); - LOG(INFO) << " create barrierEndStat: " << name - << " endBarrier warning rate: " << rateThreshold_; -} - -/* - * Note: - * the design different pserver entity owns different statSet to obey - * the background that different pserver runs separately. - */ -void BarrierEndStat::updateStat(struct timeval &cur, int32_t trainerId) { - CHECK_LT(trainerId, numConnThreads_) << "trainerId is invalid in barrier"; - - std::lock_guard guard(lock_); - timeVector_->addTimeval(cur, trainerId); - - if (timeVector_->full()) { - std::lock_guard abstractGuard(abstractLock_); - auto id = timeVector_->getLastTrainerId(); - auto delta = timeToMicroSecond(timeVector_->getDelta()); - auto secondDelta = timeToMicroSecond(timeVector_->get1NDelta()); - auto lastTwoDelta = timeToMicroSecond(timeVector_->getMinus1NDelta()); - auto midDelta = timeToMicroSecond(timeVector_->getMidNDelta()); - // discard first sample, since first sample probably is abnormal. - if (totSamples_) { - abstract_[id].freq++; - - if (delta < abstract_[id].minDelta) { - abstract_[id].minDelta = delta; - } - if (delta > abstract_[id].maxDelta) { - abstract_[id].maxDelta = delta; - } - abstract_[id].totDelta += delta; - abstract_[id].totSecondDelta += secondDelta; - abstract_[id].totLastTwoDelta += lastTwoDelta; - abstract_[id].totMidDelta += midDelta; - - // update totAbstract_ - totAbstract_.freq++; - if (delta < totAbstract_.minDelta) { - totAbstract_.minDelta = delta; - } - if (delta > totAbstract_.maxDelta) { - totAbstract_.maxDelta = delta; - } - totAbstract_.totDelta += delta; - totAbstract_.totSecondDelta += secondDelta; - totAbstract_.totLastTwoDelta += lastTwoDelta; - totAbstract_.totMidDelta += midDelta; - } - - totSamples_++; - timeVector_->reset(); - } -} - -void BarrierEndStat::reset(bool clearRawData) { - int32_t i = 0; - - totSamples_ = 0; - - std::lock_guard guard(abstractLock_); - - if (clearRawData) { - timeVector_->reset(); - } - - for (auto &abstract : abstract_) { - memset((void *)&abstract, 0, sizeof(abstract)); - abstract.minDelta = UINT64_MAX; - abstract.trainerId = i++; - } - memset((void *)&totAbstract_, 0, sizeof(Abstract)); - totAbstract_.minDelta = UINT64_MAX; -} - -void BarrierEndStat::showAbstract(std::ostream &output) const { - // do not support the case "<=2 pserver" - if (numConnThreads_ <= 2 || !totSamples_) { - return; - } - - // duplicate freq info - std::vector outputAbstract = abstract_; - std::sort(outputAbstract.begin(), - outputAbstract.end(), - [](const struct Abstract &a, const struct Abstract &b) { - return a.freq > b.freq; - }); - - auto rate = (float)outputAbstract[0].freq / (float)totSamples_; - if (rate < rateThreshold_) { - return; - } - - output << std::setw(20) << name_ << std::endl; - - /* - * Note: - * avgGap: the average delta between 1 -- n arriving trainers - * avgSecondGap: the average delta between 2 -- n arriving trainers - * avgLastTwoGap: the average delta between n-1 -- n arriving trainers - * avgMidGap: the average delta between n/2 -- n arriving trainers - * rato: samples / totSamples - * - * the stat is based on per trainer if trainer_id is set, totAbstract is - * stat based on all trainers scope. - */ - output << std::setw(42) << " " << std::setw(15) << "trainerId" - << std::setw(15) << "avgGap" << std::setw(15) << "avgSecondGap" - << std::setw(15) << "avgLastTwoGap" << std::setw(15) << "avgMidGap" - << std::setw(10) << "rate" << std::setw(10) << "samples" - << std::setw(10) << "totSamples" << std::endl; - // show totAbstract, it's valuable when lastTrainerId is even-distributed' - if (!totAbstract_.freq) return; - output << std::setw(42) << " " << std::setw(15) << "totAbstract" - << std::setw(15) << (totAbstract_.totDelta / totAbstract_.freq) * 0.001 - << std::setw(15) - << (totAbstract_.totSecondDelta / totAbstract_.freq) * 0.001 - << std::setw(15) - << (totAbstract_.totLastTwoDelta / totAbstract_.freq) * 0.001 - << std::setw(15) - << (totAbstract_.totMidDelta / totAbstract_.freq) * 0.001 - << std::setw(10) << (float)totAbstract_.freq / (float)totSamples_ - << std::setw(10) << (float)totAbstract_.freq << std::setw(10) - << (float)totSamples_ << std::endl; - - // show lastTrainerId abstract - int count = 0; - for (auto &abstract : outputAbstract) { - if (!abstract.freq || count++ >= FLAGS_log_barrier_lowest_nodes) { - break; - } - // output format control - output << std::setw(42) << " " << std::setw(15) << abstract.trainerId - << std::setw(15) << (abstract.totDelta / abstract.freq) * 0.001 - << std::setw(15) << (abstract.totSecondDelta / abstract.freq) * 0.001 - << std::setw(15) - << (abstract.totLastTwoDelta / abstract.freq) * 0.001 - << std::setw(15) << (abstract.totMidDelta / abstract.freq) * 0.001 - << std::setw(10) << (float)abstract.freq / (float)totSamples_ - << std::setw(10) << (float)abstract.freq << std::setw(10) - << (float)totSamples_ << std::endl; - } -} - -BarrierDeltaStat::BarrierDeltaStat(uint16_t numConnThreads, - const std::string &name) - : BarrierStatBase(numConnThreads, name) { - timeVector_.reset(new TimeVectorDelta(numConnThreads_)); - reset(true); - LOG(INFO) << " create barrierDeltaStat: " << name - << " barrierDelta warning rate: " << rateThreshold_; -} - -void BarrierDeltaStat::updateStat(uint64_t delta, int32_t trainerId) { - CHECK_LT(trainerId, numConnThreads_) << "trainerId is invalid in barrier"; - - std::lock_guard guard(lock_); - timeVector_->addTimeval(delta, trainerId); - - if (timeVector_->full()) { - std::lock_guard abstractGuard(abstractLock_); - auto id = timeVector_->getMaxTrainerId(); - auto delta = timeVector_->getDelta(); - // discard first sample, since first sample probably is abnormal. - if (totSamples_) { - abstract_[id].freq++; - - if (delta < abstract_[id].minDelta) { - abstract_[id].minDelta = delta; - } - if (delta > abstract_[id].maxDelta) { - abstract_[id].maxDelta = delta; - } - abstract_[id].totDelta += delta; - - // update totAbstract_ - totAbstract_.freq++; - if (delta < totAbstract_.minDelta) { - totAbstract_.minDelta = delta; - } - if (delta > totAbstract_.maxDelta) { - totAbstract_.maxDelta = delta; - } - totAbstract_.totDelta += delta; - } - - totSamples_++; - timeVector_->reset(); - } -} - -void BarrierDeltaStat::reset(bool clearRawData) { - int32_t i = 0; - - totSamples_ = 0; - - std::lock_guard guard(abstractLock_); - - if (clearRawData) { - timeVector_->reset(); - } - - for (auto &abstract : abstract_) { - memset((void *)&abstract, 0, sizeof(abstract)); - abstract.minDelta = UINT64_MAX; - abstract.trainerId = i++; - } - memset((void *)&totAbstract_, 0, sizeof(Abstract)); - totAbstract_.minDelta = UINT64_MAX; -} - -void BarrierDeltaStat::showAbstract(std::ostream &output) const { - // do not support the case "<=2 pserver" - if (numConnThreads_ <= 2 || !totSamples_) { - return; - } - - // duplicate freq info - std::vector outputAbstract = abstract_; - std::sort(outputAbstract.begin(), - outputAbstract.end(), - [](const struct Abstract &a, const struct Abstract &b) { - return a.freq > b.freq; - }); - - auto rate = (float)outputAbstract[0].freq / (float)totSamples_; - if (rate < rateThreshold_) { - return; - } - - output << std::setw(20) << name_ << std::endl; - - /* Note: - * Gap means the delta from all trainers' forwardbackward - * avgGap: average Gap in log_period batches - * minGap: min Gap in log_period batches - * maxGap: max Gap in log_period batches - * trainerId: the slowest trainer_id - * - * the stat is based on per trainer if trainer_id is set, totAbstract is - * stat based on all trainers scope. - */ - output << std::setw(42) << " " << std::setw(15) << "trainerId" - << std::setw(15) << "avgGap" << std::setw(10) << "minGap" - << std::setw(10) << "maxGap" << std::setw(10) << "rate" - << std::setw(10) << "samples" << std::setw(10) << "totSamples" - << std::endl; - // show totAbstract, it's valuable when lastTrainerId is even-distributed' - if (!totAbstract_.freq) return; - output << std::setw(42) << " " << std::setw(15) << "totAbstract" - << std::setw(15) << (totAbstract_.totDelta / totAbstract_.freq) * 0.001 - << std::setw(10) << totAbstract_.minDelta * 0.001 << std::setw(10) - << totAbstract_.maxDelta * 0.001 << std::setw(10) - << (float)totAbstract_.freq / (float)totSamples_ << std::setw(10) - << (float)totAbstract_.freq << std::setw(10) << (float)totSamples_ - << std::endl; - - // show lastTrainerId abstract - int count = 0; - for (auto &abstract : outputAbstract) { - if (!abstract.freq || count++ >= FLAGS_log_barrier_lowest_nodes) { - break; - } - // output format control - output << std::setw(42) << " " << std::setw(15) << abstract.trainerId - << std::setw(15) << (abstract.totDelta / abstract.freq) * 0.001 - << std::setw(10) << abstract.minDelta * 0.001 << std::setw(10) - << abstract.maxDelta * 0.001 << std::setw(10) - << (float)abstract.freq / (float)totSamples_ << std::setw(10) - << (float)abstract.freq << std::setw(10) << (float)totSamples_ - << std::endl; - } -} -} // namespace paddle diff --git a/paddle/utils/BarrierStat.h b/paddle/utils/BarrierStat.h deleted file mode 100644 index a9c925eff66838d58d540d7be5476e6207a30bec..0000000000000000000000000000000000000000 --- a/paddle/utils/BarrierStat.h +++ /dev/null @@ -1,425 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Locks.h" -#include "Logging.h" -#include "ThreadLocal.h" - -namespace paddle { - -inline uint64_t timeToMicroSecond(struct timeval time) { - return time.tv_sec * 1000000LU + time.tv_usec; -} - -class TimeVectorEnd { - /* - * help class for gathering all barrier performance data - * which shows time point property. - * freqently used in barrier performance tuning API, such - * as tuning which is slowest node in sync-sgd mode training. - */ -public: - explicit TimeVectorEnd(uint16_t size) : size_(size) { - index_ = 0; - timeArray_.resize(size); - trainerIds_.resize(size); - } - ~TimeVectorEnd() {} - - uint16_t size() { return size_; } - - bool full() { return index_ == size_; } - - bool empty() { return index_ == 0; } - - void reset() { index_ = 0; } - - void addTimeval(struct timeval time, int32_t trainerId) { - timeArray_[index_] = time; - trainerIds_[index_] = trainerId; - index_++; - } - - struct timeval getDelta() const { - struct timeval delta; - CHECK_GT(size_, 1) << "not support with 1 pserver"; - timersub(&timeArray_[size_ - 1], &timeArray_[0], &delta); - return delta; - } - - /* 2, n delta */ - struct timeval get1NDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - struct timeval delta; - timersub(&timeArray_[size_ - 1], &timeArray_[1], &delta); - return delta; - } - - /* n-1, n delta */ - struct timeval getMinus1NDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - struct timeval delta; - timersub(&timeArray_[size_ - 1], &timeArray_[size_ - 2], &delta); - return delta; - } - - /* n/2, n delta */ - struct timeval getMidNDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - struct timeval delta; - timersub(&timeArray_[size_ - 1], &timeArray_[size_ / 2], &delta); - return delta; - } - - int32_t getLastTrainerId() const { return trainerIds_[index_ - 1]; } - -private: - uint16_t size_; - uint16_t index_; - std::vector timeArray_; - std::vector trainerIds_; -}; - -class TimeVectorDelta { - /* - * help class for gathering performance data which shows time - * delta property, such as tuning the time distribution of - * forwardBackward time from all cluster nodes. - */ -public: - explicit TimeVectorDelta(uint16_t size) - : size_(size), min_(UINT64_MAX), max_(0) { - index_ = 0; - timeArray_.resize(size); - } - ~TimeVectorDelta() {} - - uint16_t size() { return size_; } - - bool full() { return index_ == size_; } - - bool empty() { return index_ == 0; } - - void reset() { - index_ = 0; - min_ = UINT64_MAX; - max_ = 0; - } - - void addTimeval(uint64_t delta, int32_t trainerId) { - timeArray_[index_] = delta; - index_++; - if (delta < min_) { - min_ = delta; - } - if (delta > max_) { - max_ = delta; - maxTrainerId_ = trainerId; - } - } - - uint64_t getDelta() const { - CHECK_GT(size_, 1) << "not support with 1 pserver"; - return max_ - min_; - } - - /* 2, n delta */ - uint64_t get1NDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - LOG(FATAL) << "Not implemented"; - } - - /* n-1, n delta */ - uint64_t getMinus1NDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - LOG(FATAL) << "Not implemented"; - } - - /* n/2, n delta */ - uint64_t getMidNDelta() const { - CHECK_GT(size_, 2) << "not support with less than 2 pservers"; - LOG(FATAL) << "Not implemented"; - } - - int32_t getMaxTrainerId() const { return maxTrainerId_; } - -private: - uint16_t size_; - uint16_t index_; - std::vector timeArray_; - -private: - uint64_t min_; - uint64_t max_; - int32_t maxTrainerId_; -}; - -// total samples stats, us -struct Abstract { - // last trainerId for barrier end, maxDelta trainerId for barrier delta - int32_t trainerId; - uint64_t minDelta; - uint64_t maxDelta; - uint64_t totDelta; - // first one is probably itself, so discard it. - uint64_t totSecondDelta; - // to confirm if last node destroy barrier performance. - uint64_t totLastTwoDelta; - // n/2-n delta - uint64_t totMidDelta; - uint64_t freq; -}; - -// barrier performance tunning stats -class BarrierStatBase { -public: - BarrierStatBase(uint16_t numConnThreads, const std::string &name); - - virtual ~BarrierStatBase() {} - - // if called at pserver end, then trainId means trainer's id. - // by default trainer does not use trainerId, so set it to -1 - virtual void updateStat(struct timeval &cur, int32_t trainerId = -1) = 0; - virtual void updateStat(uint64_t delta, int32_t trainerId = -1) = 0; - - const std::string &getName() { return name_; } - - virtual void reset(bool clearRawData = true) {} - // since the timeVector_ is not stateful, so it's not clear whether the - // the barrier delta is correct. if one timestamp was lost, the all data - // from barrier stat becomes rubbish. -_- - virtual bool checkPassBarrier() { - LOG(INFO) << "bug implementation found"; - return false; - } - -protected: - virtual void showAbstract(std::ostream &output) const {} - friend std::ostream &operator<<(std::ostream &output, - const BarrierStatBase &stat); - -protected: - mutable std::mutex lock_; - std::mutex abstractLock_; // see note on updaterStat - // each freqency for each barrier trainer - std::vector abstract_; - // it is valuable when do perf-tuining, if lastTrainerId acts uniform - // distribution - struct Abstract totAbstract_; - uint64_t totSamples_; - -protected: - uint16_t numConnThreads_; // total updates needed - float rateThreshold_; - std::string name_; -}; - -// the end-time of arriving real/forged barrier position -class BarrierEndStat : public BarrierStatBase { -public: - BarrierEndStat(uint16_t numConnThreads, const std::string &name); - ~BarrierEndStat() {} - - virtual void updateStat(struct timeval &cur, int32_t trainerId = -1); - virtual void updateStat(uint64_t delta, int32_t trainerId = -1) { - LOG(INFO) << "have no delta updateStat in BarrierEndStat"; - } - virtual void reset(bool clearRawData = true); - virtual bool checkPassBarrier() { return timeVector_->empty(); } - -protected: - /* - * LOG: - * readAllBlocks_denseUpdater - * trainerId avgGap avgSecondGap avgLastTwoGap avgMidGap rate - * 44 86.702 81.022 9.984 50.472 0.144737 - * 46 87.723 82.939 8.737 50.019 0.118421 - * 35 100.923 96.752 14.305 61.979 - * 0.0657895 - * log_barrier_abstract, log_barrier_lowest_nodes, log_barrier_threshold - * control details. - */ - virtual void showAbstract(std::ostream &output) const; - -private: - std::unique_ptr timeVector_; -}; - -// the delta-time from different trainers, -// eg, find the degree of imbalance of BP time at pserver end -// the entry value in timerVector_ is BP delta, do evaluation to BP delta. -class BarrierDeltaStat : public BarrierStatBase { -public: - BarrierDeltaStat(uint16_t numConnThreads, const std::string &name); - ~BarrierDeltaStat() {} - - virtual void updateStat(uint64_t delta, int32_t trainerId = -1); - virtual void updateStat(struct timeval &cur, int32_t trainerId = -1) { - LOG(INFO) << "have no timeval updateStat in BarrierDeltaStat"; - } - - virtual void reset(bool clearRawData = true); - - virtual bool checkPassBarrier() { return timeVector_->empty(); } - -protected: - virtual void showAbstract(std::ostream &outPut) const; - -private: - // store delta time in uint64_t, eg BP time of all trainers - std::unique_ptr timeVector_; -}; - -// to distinguish different contexts for same parallel threads, and different -// threads with same code-sgement, just use tagName to tag the run-time -// position. -// in Sparse, sendParallel threads can not only run in the stage of push&pull -// with same thread group, but also run in the stage of pull&push with different -// thread group, tag will be used to distinguish different run-time barrier -// position. -// trainerId in REGISTER_BARRIER_TIMER_SERVER is used to retreive lowest trainer -// nodes. - -// end barrier -#define __REGISTER_BARRIER_TIMER_SERVER( \ - set, statName, numConnThreads, trainerId, ...) \ - do { \ - if (numConnThreads > 2) { \ - std::string internalName = \ - std::string(statName) + std::string(__VA_ARGS__); \ - BarrierStatPtr __stat = \ - (set).getStat(numConnThreads, internalName, BARRIER_END); \ - struct timeval cur; \ - gettimeofday(&cur, nullptr); \ - __stat->updateStat(cur, trainerId); \ - } \ - } while (0); - -// end barrier with user-defined timer -#define __REGISTER_BARRIER_TIMER_SERVER_SET( \ - set, statName, numConnThreads, trainerId, cur, ...) \ - do { \ - if (numConnThreads > 2) { \ - std::string internalName = \ - std::string(statName) + std::string(__VA_ARGS__); \ - BarrierStatPtr __stat = \ - (set).getStat(numConnThreads, internalName, BARRIER_END); \ - __stat->updateStat(cur, trainerId); \ - } \ - } while (0); - -// delta barrier -#define __REGISTER_BARRIER_DELTA_SERVER_SET( \ - set, statName, numConnThreads, trainerId, delta, ...) \ - do { \ - if (numConnThreads > 2) { \ - std::string internalName = \ - std::string(statName) + std::string(__VA_ARGS__); \ - BarrierStatPtr __stat = \ - (set).getStat(numConnThreads, internalName, BARRIER_DELTA); \ - __stat->updateStat(delta, trainerId); \ - } \ - } while (0); - -// check end barrier -#define __CHECK_BARRIER_TIMER(set, statName, numConnThreads, ...) \ - do { \ - std::string internalName = \ - std::string(statName) + std::string(__VA_ARGS__); \ - BarrierStatPtr __stat = \ - (set).getStat(numConnThreads, internalName, BARRIER_END); \ - PCHECK(__stat->checkPassBarrier()) << internalName \ - << ": invalid barrier data"; \ - } while (0); - -/* - * Note: - * with sync-sgd algriothm in cluster mode, lots of synchronize action exsit at - * pserve end. these synchronizaton actions have impact on the efficiency of - * parameter exchange. the synchronizaton(barrier) GAP is composed of lots of - * factors, such as the forwardBackward variance, network fluncation. we try - * to have a quantitative analysis on these factor, so we design lots of barrier - * time to capture these performance. these barrier also can be placed at - * implict barrier position. - * - * example: - * in sync-sgd algorithm, each parameter server waits for all gradients from - * all trainers, thus, an explict barrier point exsit before doing optimization. - * the barrier timer located before the point can sense the barrier condition. - * - */ - -// try to capture which trainer is slowest node in sync-sgd at pserver. -#define REGISTER_SLOW_NODES_PROBE( \ - set, statName, numConnThreads, trainerId, ...) \ - __REGISTER_BARRIER_TIMER_SERVER( \ - (set), statName, numConnThreads, trainerId, __VA_ARGS__) -// try to check if all threads or trainers have passed barriers for data -// accuracy. -#define CHECK_BARRIER_TIMER(set, statName, numConnThreads, ...) \ - __CHECK_BARRIER_TIMER((set), statName, numConnThreads, __VA_ARGS__) - -#ifdef PADDLE_DISABLE_TIMER - -#define REGISTER_BARRIER_TIMER_SERVER( \ - set, statName, numConnThreads, trainerId, ...) -#define REGISTER_BARRIER_TIMER_SERVER_SET( \ - set, statName, numConnThreads, trainerId, cur, ...) -#define REGISTER_BARRIER_DELTA_SERVER_SET( \ - set, statName, numConnThreads, trainerId, cur, ...) - -#else - -/* - * sensing barrier time distribution for all parallelization threads. - * it provides low API for slow node check(REGISTER_SLOW_NODES_PROBE) - */ -#define REGISTER_BARRIER_TIMER_SERVER( \ - set, statName, numConnThreads, trainerId, ...) \ - __REGISTER_BARRIER_TIMER_SERVER( \ - (set), statName, numConnThreads, trainerId, __VA_ARGS__) - -/* - * sensing barrier time distribution for all parallelization threads. - * but time point for barrier performance is set by user. - * eg, with this api, you can get implict barrier point such as the beginning - * time distribution - * for receiving data. - */ -#define REGISTER_BARRIER_TIMER_SERVER_SET( \ - set, statName, numConnThreads, trainerId, cur, ...) \ - __REGISTER_BARRIER_TIMER_SERVER_SET( \ - (set), statName, numConnThreads, trainerId, cur, __VA_ARGS__) - -// try to capture time delta from all trainers, such as forwardBackward time -// which implies -// computation fluctuation -#define REGISTER_BARRIER_DELTA_SERVER_SET( \ - set, statName, numConnThreads, trainerId, delta, ...) \ - __REGISTER_BARRIER_DELTA_SERVER_SET( \ - (set), statName, numConnThreads, trainerId, delta, __VA_ARGS__) - -#endif // DISABLE_TIMER -} // namespace paddle diff --git a/paddle/utils/Stat.cpp b/paddle/utils/Stat.cpp index c7194d3bf1271a8bf05379f78adfb18f0f64db29..ff1b1bf888f3915f14752cb89115f7c9ed98d67f 100644 --- a/paddle/utils/Stat.cpp +++ b/paddle/utils/Stat.cpp @@ -97,34 +97,6 @@ std::ostream& operator<<(std::ostream& outPut, const Stat& stat) { return outPut; } -BarrierStatPtr StatSet::getStat(uint16_t numConnThreads, - const std::string& name, - BarrierStatType bType) { - { - ReadLockGuard guard(lock_); - auto it = barrierStatSet_.find(name); - if (it != barrierStatSet_.end()) { - return it->second; - } - } - - std::lock_guard guard(lock_); - // test again with lock_guard - auto it = barrierStatSet_.find(name); - if (it != barrierStatSet_.end()) { - return it->second; - } - - BarrierStatPtr stat; - if (bType == BARRIER_END) { - stat = std::make_shared(numConnThreads, name); - } else if (bType == BARRIER_DELTA) { - stat = std::make_shared(numConnThreads, name); - } - auto ret = barrierStatSet_.insert(std::make_pair(name, stat)); - return ret.first->second; -} - void StatSet::printSegTimerStatus() { ReadLockGuard guard(lock_); LOG(INFO) << std::setiosflags(std::ios::left) << std::setfill(' ') @@ -135,46 +107,20 @@ void StatSet::printSegTimerStatus() { } } -void StatSet::printBarrierTimerStatus() { - ReadLockGuard guard(lock_); - if (barrierStatSet_.empty()) { - return; - } - // control barrierAbstact in runtime, so enable compliation - LOG(INFO) << std::setiosflags(std::ios::left) << std::setfill(' ') - << "======= BarrierStatSet status ======" << std::endl; - for (auto& stat : barrierStatSet_) { - LOG(INFO) << std::setiosflags(std::ios::left) << std::setfill(' ') - << *(stat.second); - } -} - void StatSet::printAllStatus() { #ifndef PADDLE_DISABLE_TIMER printSegTimerStatus(); #endif - printBarrierTimerStatus(); LOG(INFO) << std::setiosflags(std::ios::left) << "--------------------------------------------------" << std::endl; } -void StatSet::printStatus(const std::string& name) { - ReadLockGuard guard(lock_); - auto iter = statSet_.find(name); - CHECK(iter != statSet_.end()) << name << " is not registed in " << name_; - LOG(INFO) << *(iter->second); -} - void StatSet::reset(bool clearRawData) { ReadLockGuard guard(lock_); for (auto& stat : statSet_) { stat.second->reset(); } - // reset barrierStat - for (auto& stat : barrierStatSet_) { - stat.second->reset(clearRawData); - } } void StatSet::setThreadInfo(const std::string& name, bool flag) { @@ -184,13 +130,6 @@ void StatSet::setThreadInfo(const std::string& name, bool flag) { iter->second->setThreadInfo(flag); } -void StatSet::deleteStat(const std::string& name) { - std::lock_guard guard(lock_); - auto iter = statSet_.find(name); - CHECK(iter != statSet_.end()) << name << " is not registed in " << name_; - statSet_.erase(iter); -} - StatInfo::~StatInfo() { if (stat_) { std::lock_guard guard(stat_->lock_); diff --git a/paddle/utils/Stat.h b/paddle/utils/Stat.h index d9cc6e413a7415d9f42508019d435992b93cf47f..79fd3b8cf043e62922dfd046754ee8ac261990c5 100644 --- a/paddle/utils/Stat.h +++ b/paddle/utils/Stat.h @@ -23,7 +23,6 @@ limitations under the License. */ #include #include -#include "BarrierStat.h" #include "Locks.h" #include "Logging.h" #include "ThreadLocal.h" @@ -60,12 +59,6 @@ public: class Stat; typedef std::shared_ptr StatPtr; -typedef std::shared_ptr BarrierStatPtr; - -enum BarrierStatType { - BARRIER_END = 0, - BARRIER_DELTA = 1, -}; class StatSet { public: @@ -74,11 +67,8 @@ public: // print to LOG(INFO) void printSegTimerStatus(); - void printBarrierTimerStatus(); void printAllStatus(); - void printStatus(const std::string& name); - StatPtr getStat(const std::string& name) { { ReadLockGuard guard(lock_); @@ -93,12 +83,6 @@ public: return ret.first->second; } - BarrierStatPtr getStat(uint16_t numConnThreads, - const std::string& name, - BarrierStatType bType); - - void deleteStat(const std::string& name); - // true for showing stats for each thread // false for showing stats aggragated over threads void setThreadInfo(const std::string& name, bool flag); @@ -120,7 +104,6 @@ public: private: std::unordered_map statSet_; - std::unordered_map barrierStatSet_; const std::string name_; RWLock lock_; }; diff --git a/paddle/utils/ThreadLocal.h b/paddle/utils/ThreadLocal.h index a4987c9ec261a2ee57e62d1640e2a21c7f804c99..b5e2862546212041a774599ec664b95e56224a07 100644 --- a/paddle/utils/ThreadLocal.h +++ b/paddle/utils/ThreadLocal.h @@ -51,7 +51,7 @@ template class ThreadLocal { public: ThreadLocal() { - PCHECK(pthread_key_create(&threadSpecificKey_, dataDestructor) == 0); + CHECK(pthread_key_create(&threadSpecificKey_, dataDestructor) == 0); } ~ThreadLocal() { pthread_key_delete(threadSpecificKey_); } @@ -65,7 +65,7 @@ public: if (!p && createLocal) { p = new T(); int ret = pthread_setspecific(threadSpecificKey_, p); - PCHECK(ret == 0); + CHECK(ret == 0); } return p; } @@ -79,7 +79,7 @@ public: if (T* q = get(false)) { dataDestructor(q); } - PCHECK(pthread_setspecific(threadSpecificKey_, p) == 0); + CHECK(pthread_setspecific(threadSpecificKey_, p) == 0); } /** @@ -112,7 +112,7 @@ private: template class ThreadLocalD { public: - ThreadLocalD() { PCHECK(pthread_key_create(&threadSpecificKey_, NULL) == 0); } + ThreadLocalD() { CHECK(pthread_key_create(&threadSpecificKey_, NULL) == 0); } ~ThreadLocalD() { pthread_key_delete(threadSpecificKey_); for (auto t : threadMap_) { @@ -127,7 +127,7 @@ public: T* p = (T*)pthread_getspecific(threadSpecificKey_); if (!p) { p = new T(); - PCHECK(pthread_setspecific(threadSpecificKey_, p) == 0); + CHECK(pthread_setspecific(threadSpecificKey_, p) == 0); updateMap(p); } return p; @@ -141,7 +141,7 @@ public: if (T* q = (T*)pthread_getspecific(threadSpecificKey_)) { dataDestructor(q); } - PCHECK(pthread_setspecific(threadSpecificKey_, p) == 0); + CHECK(pthread_setspecific(threadSpecificKey_, p) == 0); updateMap(p); } diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index ebe4f5cbb569ff37a46eb44de6362a7df337fe38..37cd16c79890738f6d8966579e15686c653d4df3 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -266,6 +266,29 @@ message PadConfig { repeated uint32 pad_w = 4; } +message MultiBoxLossConfig { + required uint32 num_classes = 1; + required float overlap_threshold = 2; + required float neg_pos_ratio = 3; + required float neg_overlap = 4; + required uint32 background_id = 5; + required uint32 input_num = 6; + optional uint32 height = 7 [default = 1]; + optional uint32 width = 8 [default = 1]; +} + +message DetectionOutputConfig { + required uint32 num_classes = 1; + required float nms_threshold = 2; + required uint32 nms_top_k = 3; + required uint32 background_id = 4; + required uint32 input_num = 5; + required uint32 keep_top_k = 6; + required float confidence_threshold = 7; + optional uint32 height = 8 [default = 1]; + optional uint32 width = 9 [default = 1]; +} + message LayerInputConfig { required string input_layer_name = 1; optional string input_parameter_name = 2; @@ -284,6 +307,8 @@ message LayerInputConfig { optional PriorBoxConfig priorbox_conf = 13; optional PadConfig pad_conf = 14; optional RowConvConfig row_conv_conf = 15; + optional MultiBoxLossConfig multibox_loss_conf = 16; + optional DetectionOutputConfig detection_output_conf = 17; } message LayerConfig { diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 361e764e25ba1801bd22f785bc282e51f058aae6..13a1802ee3790b1255fc11f5b2053e3342c61914 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -29,7 +29,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp - DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) + DEPENDS gen_proto_py framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) @@ -43,6 +43,7 @@ if (WITH_TESTING) add_subdirectory(paddle/v2/tests) add_subdirectory(paddle/v2/reader/tests) add_subdirectory(paddle/v2/plot/tests) + add_subdirectory(paddle/v2/framework/tests) endif() endif() install(DIRECTORY ${PADDLE_PYTHON_PACKAGE_DIR} diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b7418101d83fde1b91781d3a42b056cc7708cba9..826ba2834a820d11e69feec5569ef3537194e3c3 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1353,7 +1353,8 @@ class LayerBase(object): device=None, active_type="", drop_rate=0., - coeff=None): + coeff=None, + error_clipping_threshold=None): config_assert('@' not in name, "layer name: %s contain special character @" % name) global g_current_submodel @@ -1387,6 +1388,9 @@ class LayerBase(object): elif g_default_device is not None: self.config.device = g_default_device + if error_clipping_threshold is not None: + self.config.error_clipping_threshold = error_clipping_threshold + for input_index in xrange(len(self.inputs)): input = self.inputs[input_index] input_config = None @@ -1674,6 +1678,52 @@ class PriorBoxLayer(LayerBase): self.config.size = size +@config_layer('multibox_loss') +class MultiBoxLossLayer(LayerBase): + def __init__(self, name, inputs, input_num, num_classes, overlap_threshold, + neg_pos_ratio, neg_overlap, background_id, **xargs): + super(MultiBoxLossLayer, self).__init__(name, 'multibox_loss', 0, + inputs) + config_assert( + len(inputs) == (input_num * 2 + 2), + 'MultiBoxLossLayer does not have enough inputs') + config_assert(num_classes > background_id, + 'Classes number must greater than background ID') + self.config.inputs[0].multibox_loss_conf.num_classes = num_classes + self.config.inputs[ + 0].multibox_loss_conf.overlap_threshold = overlap_threshold + self.config.inputs[0].multibox_loss_conf.neg_pos_ratio = neg_pos_ratio + self.config.inputs[0].multibox_loss_conf.neg_overlap = neg_overlap + self.config.inputs[0].multibox_loss_conf.background_id = background_id + self.config.inputs[0].multibox_loss_conf.input_num = input_num + self.config.size = 1 + + +@config_layer('detection_output') +class DetectionOutputLayer(LayerBase): + def __init__(self, name, inputs, size, input_num, num_classes, + nms_threshold, nms_top_k, keep_top_k, confidence_threshold, + background_id, **xargs): + super(DetectionOutputLayer, self).__init__(name, 'detection_output', 0, + inputs) + config_assert( + len(inputs) == (input_num * 2 + 1), + 'DetectionOutputLayer does not have enough inputs') + config_assert(num_classes > background_id, + 'Classes number must greater than background ID') + self.config.inputs[0].detection_output_conf.num_classes = num_classes + self.config.inputs[ + 0].detection_output_conf.nms_threshold = nms_threshold + self.config.inputs[0].detection_output_conf.nms_top_k = nms_top_k + self.config.inputs[0].detection_output_conf.keep_top_k = keep_top_k + self.config.inputs[ + 0].detection_output_conf.confidence_threshold = confidence_threshold + self.config.inputs[ + 0].detection_output_conf.background_id = background_id + self.config.inputs[0].detection_output_conf.input_num = input_num + self.config.size = size + + @config_layer('data') class DataLayer(LayerBase): def __init__(self, name, size, height=None, width=None, device=None): @@ -2420,10 +2470,14 @@ class MaxLayer(LayerBase): trans_type='non-seq', bias=False, output_max_index=None, + stride=-1, **xargs): super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, **xargs) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') + if trans_type == 'seq': + config_assert(stride == -1, 'subseq does not support stride window') self.config.trans_type = trans_type + self.config.seq_pool_stride = stride for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2685,11 +2739,15 @@ class AverageLayer(LayerBase): average_strategy='average', trans_type='non-seq', bias=False, + stride=-1, **xargs): super(AverageLayer, self).__init__( name, 'average', 0, inputs=inputs, **xargs) self.config.average_strategy = average_strategy + if trans_type == 'seq': + config_assert(stride == -1, 'subseq does not support stride window') self.config.trans_type = trans_type + self.config.seq_pool_stride = stride config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -2728,13 +2786,7 @@ class TensorLayer(LayerBase): @config_layer('mixed') class MixedLayer(LayerBase): - def __init__(self, - name, - inputs, - size=0, - bias=True, - error_clipping_threshold=None, - **xargs): + def __init__(self, name, inputs, size=0, bias=True, **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) @@ -2816,9 +2868,6 @@ class MixedLayer(LayerBase): self.config.bias_size = psize self.create_bias_parameter(bias, psize) - if error_clipping_threshold is not None: - self.config.error_clipping_threshold = error_clipping_threshold - # like MixedLayer, but no bias parameter @config_func diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index a601d5c84ad222785e68b9fa81c51b1e120b4f29..0a5dd49bb48c25f268aa273314f92c092305664a 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -115,6 +115,8 @@ __all__ = [ 'print_layer', 'priorbox_layer', 'cross_channel_norm_layer', + 'multibox_loss_layer', + 'detection_output_layer', 'spp_layer', 'pad_layer', 'eos_layer', @@ -195,6 +197,8 @@ class LayerType(object): PRINT_LAYER = 'print' PRIORBOX_LAYER = 'priorbox' + MULTIBOX_LOSS_LAYER = 'multibox_loss' + DETECTION_OUTPUT_LAYER = 'detection_output' CTC_LAYER = 'ctc' WARP_CTC_LAYER = 'warp_ctc' @@ -1041,6 +1045,158 @@ def priorbox_layer(input, size=size) +@wrap_name_default("multibox_loss") +def multibox_loss_layer(input_loc, + input_conf, + priorbox, + label, + num_classes, + overlap_threshold=0.5, + neg_pos_ratio=3.0, + neg_overlap=0.5, + background_id=0, + name=None): + """ + Compute the location loss and the confidence loss for ssd. + + :param name: The Layer Name. + :type name: basestring + :param input_loc: The input predict locations. + :type input_loc: LayerOutput | List of LayerOutput + :param input_conf: The input priorbox confidence. + :type input_conf: LayerOutput | List of LayerOutput + :param priorbox: The input priorbox location and the variance. + :type priorbox: LayerOutput + :param label: The input label. + :type label: LayerOutput + :param num_classes: The number of the classification. + :type num_classes: int + :param overlap_threshold: The threshold of the overlap. + :type overlap_threshold: float + :param neg_pos_ratio: The ratio of the negative bbox to the positive bbox. + :type neg_pos_ratio: float + :param neg_overlap: The negative bbox overlap threshold. + :type neg_overlap: float + :param background_id: The background class index. + :type background_id: int + :return: LayerOutput + """ + if isinstance(input_loc, LayerOutput): + input_loc = [input_loc] + assert isinstance(input_loc, collections.Sequence) # list or tuple + for each in input_loc: + assert isinstance(each, LayerOutput) + input_loc_num = len(input_loc) + + if isinstance(input_conf, LayerOutput): + input_conf = [input_conf] + assert isinstance(input_conf, collections.Sequence) # list or tuple + for each in input_conf: + assert isinstance(each, LayerOutput) + input_conf_num = len(input_conf) + # Check the input layer number. + assert input_loc_num == input_conf_num + + inputs = [priorbox.name, label.name] + inputs.extend([l.name for l in input_loc]) + inputs.extend([l.name for l in input_conf]) + parents = [priorbox, label] + parents.extend(input_loc) + parents.extend(input_conf) + + Layer( + name=name, + type=LayerType.MULTIBOX_LOSS_LAYER, + inputs=inputs, + input_num=input_loc_num, + num_classes=num_classes, + overlap_threshold=overlap_threshold, + neg_pos_ratio=neg_pos_ratio, + neg_overlap=neg_overlap, + background_id=background_id) + return LayerOutput( + name, LayerType.MULTIBOX_LOSS_LAYER, parents=parents, size=1) + + +@wrap_name_default("detection_output") +def detection_output_layer(input_loc, + input_conf, + priorbox, + num_classes, + nms_threshold=0.45, + nms_top_k=400, + keep_top_k=200, + confidence_threshold=0.01, + background_id=0, + name=None): + """ + Apply the NMS to the output of network and compute the predict bounding + box location. + + :param name: The Layer Name. + :type name: basestring + :param input_loc: The input predict locations. + :type input_loc: LayerOutput | List of LayerOutput. + :param input_conf: The input priorbox confidence. + :type input_conf: LayerOutput | List of LayerOutput. + :param priorbox: The input priorbox location and the variance. + :type priorbox: LayerOutput + :param num_classes: The number of the classification. + :type num_classes: int + :param nms_threshold: The Non-maximum suppression threshold. + :type nms_threshold: float + :param nms_top_k: The bbox number kept of the NMS's output + :type nms_top_k: int + :param keep_top_k: The bbox number kept of the layer's output + :type keep_top_k: int + :param confidence_threshold: The classification confidence threshold + :type confidence_threshold: float + :param background_id: The background class index. + :type background_id: int + :return: LayerOutput + """ + if isinstance(input_loc, LayerOutput): + input_loc = [input_loc] + assert isinstance(input_loc, collections.Sequence) # list or tuple + for each in input_loc: + assert isinstance(each, LayerOutput) + input_loc_num = len(input_loc) + + if isinstance(input_conf, LayerOutput): + input_conf = [input_conf] + assert isinstance(input_conf, collections.Sequence) # list or tuple + for each in input_conf: + assert isinstance(each, LayerOutput) + input_conf_num = len(input_conf) + + # Check the input layer number. + assert input_loc_num == input_conf_num + + inputs = [priorbox.name] + inputs.extend([l.name for l in input_loc]) + inputs.extend([l.name for l in input_conf]) + parents = [priorbox] + parents.extend(input_loc) + parents.extend(input_conf) + + size = keep_top_k * 7 + + Layer( + name=name, + type=LayerType.DETECTION_OUTPUT_LAYER, + inputs=inputs, + size=size, + input_num=input_loc_num, + num_classes=num_classes, + nms_threshold=nms_threshold, + nms_top_k=nms_top_k, + keep_top_k=keep_top_k, + confidence_threshold=confidence_threshold, + background_id=background_id) + return LayerOutput( + name, LayerType.DETECTION_OUTPUT_LAYER, parents=parents, size=size) + + @wrap_name_default("cross_channel_norm") def cross_channel_norm_layer(input, name=None, param_attr=None): """ @@ -1090,10 +1246,19 @@ def pooling_layer(input, name=None, bias_attr=None, agg_level=AggregateLevel.TO_NO_SEQUENCE, + stride=-1, layer_attr=None): """ Pooling layer for sequence inputs, not used for Image. + If stride > 0, this layer slides a window whose size is determined by stride, + and return the pooling value of the window as the output. Thus, a long sequence + will be shorten. + + The parameter stride specifies the intervals at which to apply the pooling + operation. Note that for sequence with sub-sequence, the default value + of stride is -1. + The example usage is: .. code-block:: python @@ -1112,6 +1277,8 @@ def pooling_layer(input, :param pooling_type: Type of pooling, MaxPooling(default), AvgPooling, SumPooling, SquareRootNPooling. :type pooling_type: BasePoolingType|None + :param stride: The step size between successive pooling regions. + :type stride: Int :param bias_attr: Bias parameter attribute. False if no bias. :type bias_attr: ParameterAttribute|None|False :param layer_attr: The Extra Attributes for layer, such as dropout. @@ -1129,12 +1296,16 @@ def pooling_layer(input, extra_dict['output_max_index'] = pooling_type.output_max_index extra_dict.update(ExtraLayerAttribute.to_kwargs(layer_attr)) + if agg_level == AggregateLevel.TO_SEQUENCE: + assert stride == -1 + Layer( name=name, type=pooling_type.name, inputs=[Input(input.name)], bias=ParamAttr.to_bias(bias_attr), trans_type=agg_level, + stride=stride, **extra_dict) return LayerOutput( @@ -1396,7 +1567,7 @@ def last_seq(input, :type name: basestring :param input: Input layer name. :type input: LayerOutput - :param stride: window size. + :param stride: The step size between successive pooling regions. :type stride: Int :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -1452,7 +1623,7 @@ def first_seq(input, :type name: basestring :param input: Input layer name. :type input: LayerOutput - :param stride: window size. + :param stride: The step size between successive pooling regions. :type stride: Int :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index c0e87d6de372dfdd9c7e694af71df8f3b011d43a..a939c41ad01922e421f7bcd93851df7447a6799f 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -6,6 +6,6 @@ img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cos test_rnn_group shared_fc shared_lstm shared_gru test_cost_layers_with_weight test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer -test_prelu_layer test_row_conv) +test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_detection_output_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_detection_output_layer.protostr new file mode 100644 index 0000000000000000000000000000000000000000..6690f9852a31b1909df7df99720db639eb2a564d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_detection_output_layer.protostr @@ -0,0 +1,66 @@ +type: "nn" +layers { + name: "input_loc" + type: "data" + size: 16 + active_type: "" + height: 16 + width: 1 +} +layers { + name: "input_conf" + type: "data" + size: 8 + active_type: "" + height: 1 + width: 8 +} +layers { + name: "priorbox" + type: "data" + size: 32 + active_type: "" + height: 4 + width: 8 +} +layers { + name: "test_detection_output" + type: "detection_output" + size: 1400 + active_type: "" + inputs { + input_layer_name: "priorbox" + detection_output_conf { + num_classes: 21 + nms_threshold: 0.45 + nms_top_k: 400 + background_id: 0 + input_num: 1 + keep_top_k: 200 + confidence_threshold: 0.01 + } + } + inputs { + input_layer_name: "input_loc" + } + inputs { + input_layer_name: "input_conf" + } +} +input_layer_names: "priorbox" +input_layer_names: "input_loc" +input_layer_names: "input_conf" +output_layer_names: "test_detection_output" +sub_models { + name: "root" + layer_names: "input_loc" + layer_names: "input_conf" + layer_names: "priorbox" + layer_names: "test_detection_output" + input_layer_names: "priorbox" + input_layer_names: "input_loc" + input_layer_names: "input_conf" + output_layer_names: "test_detection_output" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multibox_loss_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multibox_loss_layer.protostr new file mode 100644 index 0000000000000000000000000000000000000000..0ba84dcc6db6b7025a98b2698312f5fc9e0ed634 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_multibox_loss_layer.protostr @@ -0,0 +1,79 @@ +type: "nn" +layers { + name: "input_loc" + type: "data" + size: 16 + active_type: "" + height: 16 + width: 1 +} +layers { + name: "input_conf" + type: "data" + size: 8 + active_type: "" + height: 1 + width: 8 +} +layers { + name: "priorbox" + type: "data" + size: 32 + active_type: "" + height: 4 + width: 8 +} +layers { + name: "label" + type: "data" + size: 24 + active_type: "" + height: 4 + width: 6 +} +layers { + name: "test_multibox_loss" + type: "multibox_loss" + size: 1 + active_type: "" + inputs { + input_layer_name: "priorbox" + multibox_loss_conf { + num_classes: 21 + overlap_threshold: 0.5 + neg_pos_ratio: 3.0 + neg_overlap: 0.5 + background_id: 0 + input_num: 1 + } + } + inputs { + input_layer_name: "label" + } + inputs { + input_layer_name: "input_loc" + } + inputs { + input_layer_name: "input_conf" + } +} +input_layer_names: "priorbox" +input_layer_names: "label" +input_layer_names: "input_loc" +input_layer_names: "input_conf" +output_layer_names: "test_multibox_loss" +sub_models { + name: "root" + layer_names: "input_loc" + layer_names: "input_conf" + layer_names: "priorbox" + layer_names: "label" + layer_names: "test_multibox_loss" + input_layer_names: "priorbox" + input_layer_names: "label" + input_layer_names: "input_loc" + input_layer_names: "input_conf" + output_layer_names: "test_multibox_loss" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr index 5a217f5544a8a3b4704b158dfeb92f747b7bd94b..8989561df04a60c906c06432fd857227a3814194 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr @@ -14,6 +14,7 @@ layers { input_layer_name: "dat_in" } trans_type: "seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_1__" @@ -24,6 +25,7 @@ layers { input_layer_name: "dat_in" } trans_type: "non-seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_2__" @@ -35,6 +37,7 @@ layers { } average_strategy: "average" trans_type: "seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_3__" @@ -46,6 +49,7 @@ layers { } average_strategy: "average" trans_type: "non-seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_4__" @@ -57,6 +61,7 @@ layers { } average_strategy: "sum" trans_type: "seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_5__" @@ -68,6 +73,7 @@ layers { } average_strategy: "sum" trans_type: "non-seq" + seq_pool_stride: -1 } layers { name: "__seq_pooling_6__" @@ -77,8 +83,44 @@ layers { inputs { input_layer_name: "dat_in" } + trans_type: "non-seq" + seq_pool_stride: 5 +} +layers { + name: "__seq_pooling_7__" + type: "average" + size: 100 + active_type: "" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "average" + trans_type: "non-seq" + seq_pool_stride: 5 +} +layers { + name: "__seq_pooling_8__" + type: "average" + size: 100 + active_type: "" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "sum" + trans_type: "non-seq" + seq_pool_stride: 5 +} +layers { + name: "__seq_pooling_9__" + type: "max" + size: 100 + active_type: "" + inputs { + input_layer_name: "dat_in" + } output_max_index: true trans_type: "non-seq" + seq_pool_stride: -1 } input_layer_names: "dat_in" output_layer_names: "__seq_pooling_0__" @@ -88,6 +130,9 @@ output_layer_names: "__seq_pooling_3__" output_layer_names: "__seq_pooling_4__" output_layer_names: "__seq_pooling_5__" output_layer_names: "__seq_pooling_6__" +output_layer_names: "__seq_pooling_7__" +output_layer_names: "__seq_pooling_8__" +output_layer_names: "__seq_pooling_9__" sub_models { name: "root" layer_names: "dat_in" @@ -98,6 +143,9 @@ sub_models { layer_names: "__seq_pooling_4__" layer_names: "__seq_pooling_5__" layer_names: "__seq_pooling_6__" + layer_names: "__seq_pooling_7__" + layer_names: "__seq_pooling_8__" + layer_names: "__seq_pooling_9__" input_layer_names: "dat_in" output_layer_names: "__seq_pooling_0__" output_layer_names: "__seq_pooling_1__" @@ -106,6 +154,9 @@ sub_models { output_layer_names: "__seq_pooling_4__" output_layer_names: "__seq_pooling_5__" output_layer_names: "__seq_pooling_6__" + output_layer_names: "__seq_pooling_7__" + output_layer_names: "__seq_pooling_8__" + output_layer_names: "__seq_pooling_9__" is_recurrent_layer_group: false } diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..3572a2cb07d95ffaec261bdc63492ade734ea8b9 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_detection_output_layer.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +input_loc = data_layer(name='input_loc', size=16, height=16, width=1) + +input_conf = data_layer(name='input_conf', size=8, height=1, width=8) + +priorbox = data_layer(name='priorbox', size=32, height=4, width=8) + +detout = detection_output_layer( + input_loc=input_loc, + input_conf=input_conf, + priorbox=priorbox, + num_classes=21, + nms_threshold=0.45, + nms_top_k=400, + keep_top_k=200, + confidence_threshold=0.01, + background_id=0, + name='test_detection_output') + +outputs(detout) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..c3376c47bded5a3aad15331936a61e12ac883b17 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_multibox_loss_layer.py @@ -0,0 +1,25 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +input_loc = data_layer(name='input_loc', size=16, height=16, width=1) + +input_conf = data_layer(name='input_conf', size=8, height=1, width=8) + +priorbox = data_layer(name='priorbox', size=32, height=4, width=8) + +label = data_layer(name='label', size=24, height=4, width=6) + +multibox_loss = multibox_loss_layer( + input_loc=input_loc, + input_conf=input_conf, + priorbox=priorbox, + label=label, + num_classes=21, + overlap_threshold=0.5, + neg_pos_ratio=3.0, + neg_overlap=0.5, + background_id=0, + name='test_multibox_loss') + +outputs(multibox_loss) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py index 3c49eb56c1363a6a3f365fe56e16a8b484c8a004..3c205eabd80492a68383fdbecd14a7d6db3e16eb 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -14,6 +14,14 @@ for pt in POOL_TYPE: for al in AGG_LEVEL: opts.append(pooling_layer(input=din, agg_level=al, pooling_type=pt())) +for pt in POOL_TYPE: + opts.append( + pooling_layer( + input=din, + agg_level=AggregateLevel.TO_NO_SEQUENCE, + pooling_type=pt(), + stride=5)) + opts.append( pooling_layer( input=din, pooling_type=MaxPooling(output_max_index=True))) diff --git a/python/paddle/v2/framework/__init__.py b/python/paddle/v2/framework/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c942373c667733f8aabe63026998a8915618130a --- /dev/null +++ b/python/paddle/v2/framework/__init__.py @@ -0,0 +1 @@ +__all__ = ['proto'] diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..8cb0c5c3765a00b45177117925e320e61a1b609a --- /dev/null +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -0,0 +1 @@ +add_python_test(test_framework test_protobuf.py) diff --git a/python/paddle/v2/framework/tests/test_protobuf.py b/python/paddle/v2/framework/tests/test_protobuf.py new file mode 100644 index 0000000000000000000000000000000000000000..f0e60191991e2f24b0a1972afb0f6cbd3aaa4008 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_protobuf.py @@ -0,0 +1,26 @@ +import paddle.v2.framework.proto.op_proto_pb2 +import paddle.v2.framework.proto.attr_type_pb2 +import unittest + + +class TestFrameworkProto(unittest.TestCase): + def test_all(self): + op_proto_lib = paddle.v2.framework.proto.op_proto_pb2 + attr_type_lib = paddle.v2.framework.proto.attr_type_pb2 + op_proto = op_proto_lib.OpProto() + ipt0 = op_proto.inputs.add() + ipt0.name = "a" + ipt0.comment = "the input of cosine op" + ipt1 = op_proto.inputs.add() + ipt1.name = "b" + ipt1.comment = "the other input of cosine op" + opt = op_proto.outputs.add() + opt.name = "output" + opt.comment = "the output of cosine op" + op_proto.comment = "cosine op, output = scale*cos(a, b)" + attr = op_proto.attrs.add() + attr.name = "scale" + attr.comment = "scale of cosine op" + attr.type = attr_type_lib.FLOAT + op_proto.type = "cos" + self.assertTrue(op_proto.IsInitialized()) diff --git a/python/paddle/v2/master/client.py b/python/paddle/v2/master/client.py index de8e9bb88e1064e41a80e0ef7838e307089a1331..70f9e43c9683033233d48a750668771a4c7ba045 100644 --- a/python/paddle/v2/master/client.py +++ b/python/paddle/v2/master/client.py @@ -26,14 +26,22 @@ class client(object): holder[idx] = c_ptr lib.paddle_set_dataset(self.c, holder, len(paths)) + # return format: (record, errno) + # errno = 0: ok + # < 0: error def next_record(self): p = ctypes.c_char_p() ret = ctypes.pointer(p) size = lib.paddle_next_record(self.c, ret) + if size < 0: + # Error + return None, size + if size == 0: # Empty record - return "" + return "", 0 + record = ret.contents.value[:size] # Memory created from C should be freed. lib.mem_free(ret.contents) - return record + return record, 0 diff --git a/python/paddle/v2/reader/creator.py b/python/paddle/v2/reader/creator.py index 9f888b16d6b2fbf457ee4f4fe94fcb51b6f37fc9..61b5cc134fba875955bdbfddc2bb1e083241940d 100644 --- a/python/paddle/v2/reader/creator.py +++ b/python/paddle/v2/reader/creator.py @@ -57,17 +57,20 @@ def text_file(path): return reader -def recordio(path): +def recordio_local(paths, buf_size=100): """ - Creates a data reader that outputs record one one by one from given recordio file - :path: path of recordio file - :returns: data reader of recordio file + Creates a data reader from given RecordIO file paths separated by ",", + glob pattern is supported. + :path: path of recordio files. + :returns: data reader of recordio files. """ import recordio as rec + import paddle.v2.reader.decorator as dec def reader(): - f = rec.reader(path) + a = ','.join(paths) + f = rec.reader(a) while True: r = f.read() if r is None: @@ -75,4 +78,38 @@ def recordio(path): yield r f.close() + return dec.buffered(reader, buf_size) + + +def recordio(paths, buf_size=100): + """ + Creates a data reader that outputs record one one by one + from given local or cloud recordio path. + :path: path of recordio files. + :returns: data reader of recordio files. + """ + import os + import paddle.v2.master.client as cloud + + if "KUBERNETES_SERVICE_HOST" not in os.environ.keys(): + return recordio_local(paths) + + host_name = "MASTER_SERVICE_HOST" + if host_name not in os.environ.keys(): + raise Exception('not find ' + host_name + ' in environ.') + + addr = os.environ(host) + + def reader(): + c = cloud(addr, buf_size) + c.set_dataset(paths) + + while True: + r, err = client.next_record() + if err < 0: + break + yield r + + c.close() + return reader diff --git a/python/paddle/v2/reader/tests/creator_test.py b/python/paddle/v2/reader/tests/creator_test.py index ba4f558874a0155d276fcb0e0d2d9258f0903f0e..b42d273ecfe6c4bc5706ec52617960b83496d70d 100644 --- a/python/paddle/v2/reader/tests/creator_test.py +++ b/python/paddle/v2/reader/tests/creator_test.py @@ -38,7 +38,7 @@ class TestRecordIO(unittest.TestCase): def test_recordio(self): path = os.path.join( os.path.dirname(__file__), "test_recordio_creator.dat") - reader = paddle.v2.reader.creator.recordio(path) + reader = paddle.v2.reader.creator.recordio([path]) for idx, r in enumerate(reader()): self.assertSequenceEqual(r, str(idx)) diff --git a/python/setup.py.in b/python/setup.py.in index dae01664876a913b49403d3f95001f009721f73b..eeffbfe80e3b4b5473b06f7372addd9870034e77 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -9,7 +9,9 @@ packages=['paddle', 'paddle.v2.dataset', 'paddle.v2.reader', 'paddle.v2.master', - 'paddle.v2.plot'] + 'paddle.v2.plot', + 'paddle.v2.framework', + 'paddle.v2.framework.proto'] setup_requires=["requests", "numpy", @@ -27,8 +29,11 @@ setup(name='paddle', description='Parallel Distributed Deep Learning', install_requires=setup_requires, packages=packages, - package_data={'paddle.v2.master': ['${paddle_master_LIB_NAME}'], }, + package_data={'paddle.v2.master': ['libpaddle_master.so'], }, package_dir={ - '': '${CMAKE_CURRENT_SOURCE_DIR}' + '': '${CMAKE_CURRENT_SOURCE_DIR}', + # The paddle.v2.framework.proto will be generated while compiling. + # So that package points to other directory. + 'paddle.v2.framework.proto': '${CMAKE_BINARY_DIR}/paddle/framework' }, )