diff --git a/.gitignore b/.gitignore index 275173b9677bffe028152fe8eadb3384329aeb5a..c84b2fc8c79d6e2c9c83e2b830ab176295846fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,9 @@ third_party/ # clion workspace. cmake-build-* + +# generated while compiling +python/paddle/v2/framework/core.so +CMakeFiles +cmake_install.cmake + diff --git a/.travis.yml b/.travis.yml index 498674469b27f585af798b95f30b74ebed99e32c..2cf7666fb5d0c47034676864a52c3d3dbce19683 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ cache: - $HOME/.ccache - $HOME/.cache/pip - $TRAVIS_BUILD_DIR/build/third_party + - $TRAVIS_BUILD_DIR/build_android/third_party sudo: required dist: trusty os: @@ -11,6 +12,7 @@ os: env: - JOB=build_doc - JOB=check_style + - JOB=build_android addons: apt: packages: diff --git a/CMakeLists.txt b/CMakeLists.txt index 15a7c6b07417adfacd461e95c0b92f658e1e11cc..fb1c85bf742c80308edb009c080cb0da6d409ee0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,9 @@ if(NOT CMAKE_CROSSCOMPILING) endif(NOT CMAKE_CROSSCOMPILING) find_package(Git REQUIRED) find_package(Threads REQUIRED) -find_package(Boost QUIET) +if(NOT ANDROID) + find_package(Boost QUIET) +endif() include(simd) @@ -97,6 +99,7 @@ include(external/swig) # download, build, install swig include(external/warpctc) # download, build, install warpctc include(external/any) # download libn::any include(external/eigen) # download eigen3 +include(external/pybind11) # download pybind11 include(cudnn) # set cudnn libraries, must before configure include(configure) # add paddle env configuration @@ -139,6 +142,10 @@ endif(USE_NNPACK) add_subdirectory(proto) +# "add_subdirectory(go)" should be placed after the following loine, +# because it depends on paddle/optimizer. +add_subdirectory(paddle/optimizer) + # "add_subdirectory(paddle)" and "add_subdirectory(python)" should be # placed after this block, because they depends on it. if(WITH_GOLANG) @@ -146,7 +153,9 @@ if(WITH_GOLANG) endif(WITH_GOLANG) add_subdirectory(paddle) -add_subdirectory(python) +if(WITH_PYTHON) + add_subdirectory(python) +endif() if(WITH_DOC) add_subdirectory(doc) endif() diff --git a/cmake/cross_compiling/android.cmake b/cmake/cross_compiling/android.cmake index 9724c16122ab2e6be55864c8716698c9b9d7c3f0..dcfbc5d0129d7763daaf17c33d2b7791e87d3018 100644 --- a/cmake/cross_compiling/android.cmake +++ b/cmake/cross_compiling/android.cmake @@ -106,6 +106,9 @@ IF("${CMAKE_VERSION}" VERSION_LESS "3.7.0") SET(CMAKE_SYSTEM_PROCESSOR armv7-a) ENDIF() ENDIF() + IF(ANDROID_ABI STREQUAL "arm64-v8a") + SET(ANDROID_TOOLCHAIN_NAME aarch64-linux-android) + ENDIF() SET(ANDROID_TOOLCHAIN_PREFIX "${ANDROID_TOOLCHAIN_ROOT}/bin/${ANDROID_TOOLCHAIN_NAME}-") ENDIF() @@ -162,6 +165,10 @@ IF("${CMAKE_VERSION}" VERSION_LESS "3.7.0") ENDIF() ENDIF() + IF(ANDROID_ABI STREQUAL "arm64-v8a") + LIST(APPEND ANDROID_COMPILER_FLAGS -march=armv8-a) + ENDIF() + STRING(REPLACE ";" " " ANDROID_COMPILER_FLAGS "${ANDROID_COMPILER_FLAGS}") STRING(REPLACE ";" " " ANDROID_LINKER_FLAGS "${ANDROID_LINKER_FLAGS}") diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index bd401faa6eb8a583bce542db68852f8571681daf..8a594a825abdca6a0f989b94fa42f97d6df5e10a 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -52,6 +52,7 @@ ExternalProject_Add( ADD_LIBRARY(glog STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES}) -ADD_DEPENDENCIES(glog extern_glog) +ADD_DEPENDENCIES(glog extern_glog gflags) +LINK_LIBRARIES(glog gflags) LIST(APPEND external_project_dependencies glog) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 5b9d9844ed21ceb507a8e01676c3533f4e3dd8fb..60a1041936437775e0994157b8ffcb7c52b7ab87 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -32,7 +32,12 @@ IF(NOT ${CBLAS_FOUND}) # arm_soft_fp_abi branch of OpenBLAS to support softfp # https://github.com/xianyi/OpenBLAS/tree/arm_soft_fp_abi SET(OPENBLAS_COMMIT "b5c96fcfcdc82945502a2303116a64d89985daf5") - SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=ARMV7 ARM_SOFTFP_ABI=1 USE_THREAD=0) + IF(ANDROID_ABI MATCHES "^armeabi(-v7a)?$") + SET(TARGET "ARMV7") + ELSEIF(ANDROID_ABI STREQUAL "arm64-v8a") + SET(TARGET "ARMV8") + ENDIF() + SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=${TARGET} ARM_SOFTFP_ABI=1 USE_THREAD=0) ELSEIF(RPI) # use hardfp SET(OPENBLAS_COMMIT "v0.2.19") diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake new file mode 100644 index 0000000000000000000000000000000000000000..9391c285c7544669a5b1a078b7473d7a656c1bb4 --- /dev/null +++ b/cmake/external/pybind11.cmake @@ -0,0 +1,30 @@ +INCLUDE(ExternalProject) + +SET(PYBIND_SOURCE_DIR ${THIRD_PARTY_PATH}/pybind) + +INCLUDE_DIRECTORIES(${PYBIND_SOURCE_DIR}/src/extern_pybind/include) + +ExternalProject_Add( + extern_pybind + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY "https://github.com/pybind/pybind11.git" + GIT_TAG "v2.1.1" + PREFIX ${PYBIND_SOURCE_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) + +if (${CMAKE_VERSION} VERSION_LESS "3.3.0") + set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/pybind_dummy.c) + file(WRITE ${dummyfile} "const char * dummy_any = \"${dummyfile}\";") + add_library(pybind STATIC ${dummyfile}) +else() + add_library(pybind INTERFACE) +endif() + +add_dependencies(pybind extern_pybind) + +LIST(APPEND external_project_dependencies pybind) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 6546b2c83bc8f81f89e4018a2216f191bbeb0d21..67a359d4b5f4cca8fc8e74eab4d4acb4cc12baed 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -18,6 +18,9 @@ INCLUDE(python_module) FIND_PACKAGE(PythonInterp 2.7) IF(WITH_PYTHON) FIND_PACKAGE(PythonLibs 2.7) + # Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE. + ADD_LIBRARY(python SHARED IMPORTED GLOBAL) + SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES}) ENDIF(WITH_PYTHON) SET(py_env "") diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 7a996dea92b13bdac054a987a004a3d54ff02da2..c31e62fc08b531a38a851b71a033e14277eff015 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -109,7 +109,9 @@ set(COMMON_FLAGS -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare - -Wno-error=unused-local-typedefs) + -Wno-error=unused-local-typedefs + -Wno-error=parentheses-equality # Warnings in Pybind11 +) set(GPU_COMMON_FLAGS -fPIC diff --git a/cmake/generic.cmake b/cmake/generic.cmake index f568b23c8a2aad24ef58645f3601274dfdc7d01a..48c054d17fa0b06998bd6b2a2a2b324afd2439e9 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -90,11 +90,11 @@ # including binary directory for generated headers. include_directories(${CMAKE_CURRENT_BINARY_DIR}) -if(NOT APPLE) +if(NOT APPLE AND NOT ANDROID) find_package(Threads REQUIRED) link_libraries(${CMAKE_THREAD_LIBS_INIT}) - set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -ldl") -endif(NOT APPLE) + set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -ldl -lrt") +endif(NOT APPLE AND NOT ANDROID) function(merge_static_libs TARGET_NAME) set(libs ${ARGN}) @@ -302,7 +302,7 @@ function(go_library TARGET_NAME) file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go") string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - # FIXME: link path + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND rm "${${TARGET_NAME}_LIB_PATH}" # Golang build source code @@ -310,7 +310,7 @@ function(go_library TARGET_NAME) -o "${${TARGET_NAME}_LIB_PATH}" "./${CMAKE_CURRENT_SOURCE_REL_DIR}/${GO_SOURCE}" # must run under GOPATH - WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") + WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") add_dependencies(${TARGET_NAME} go_vendor) endfunction(go_library) @@ -321,14 +321,11 @@ function(go_binary TARGET_NAME) cmake_parse_arguments(go_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - # FIXME: link path add_custom_command(OUTPUT ${TARGET_NAME}_timestamp - COMMAND env LIBRARY_PATH=${CMAKE_BINARY_DIR}/go/pserver/client/c/:$ENV{LIBRARY_PATH} - GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build + COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}" "./${CMAKE_CURRENT_SOURCE_REL_DIR}/${go_binary_SRCS}" WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") - # TODO: don't know what ${TARGET_NAME}_link does add_custom_target(${TARGET_NAME} ALL DEPENDS go_vendor ${TARGET_NAME}_timestamp ${go_binary_DEPS}) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin) endfunction(go_binary) @@ -336,15 +333,18 @@ endfunction(go_binary) function(go_test TARGET_NAME) set(options OPTIONAL) set(oneValueArgs "") - set(multiValueArgs SRCS DEPS) + set(multiValueArgs DEPS) cmake_parse_arguments(go_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - add_custom_command(OUTPUT ${TARGET_NAME}_timestamp + string(REPLACE "${PADDLE_GO_PATH}" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + add_custom_target(${TARGET_NAME} ALL DEPENDS go_vendor ${go_test_DEPS}) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} test -c -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}" - ${go_test_SRCS} + ".${CMAKE_CURRENT_SOURCE_REL_DIR}" + WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go") + add_test(NAME ${TARGET_NAME} + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS}) - add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}) endfunction(go_test) function(proto_library TARGET_NAME) diff --git a/doc/howto/dev/new_layer_cn.rst b/doc/howto/dev/new_layer_cn.rst index 9489a921c70ad6ee5709f46445554f5d9640162c..75037e693b32f923ee7dc9dfec322495fe4ce10a 100644 --- a/doc/howto/dev/new_layer_cn.rst +++ b/doc/howto/dev/new_layer_cn.rst @@ -37,7 +37,7 @@ \frac{\partial c(y)}{\partial x} = \frac{\partial c(y)}{\partial y} \frac{\partial y}{\partial x} -假设 :math:`z = f(W^T x + b)` ，那么 +假设 :math:`z = W^T x + b` ，那么 .. math:: diff --git a/doc/howto/dev/new_layer_en.rst b/doc/howto/dev/new_layer_en.rst index 46481f5ead33dc6a26507e021fd9ae0f8316e940..110a9fb38f890a766bb4480e91feb22d3b0838a5 100644 --- a/doc/howto/dev/new_layer_en.rst +++ b/doc/howto/dev/new_layer_en.rst @@ -29,7 +29,7 @@ Fully connected layer takes a dense input vector with dimension :math:`D_i`. It where :math:`f(.)` is an nonlinear *activation* function, such as sigmoid, tanh, and Relu. -The transformation matrix :math:`W` and bias vector :math:`b` are the *parameters* of the layer. The *parameters* of a layer are learned during training in the *backward pass*. The backward pass computes the gradients of the output function with respect to all parameters and inputs. The optimizer can use chain rule to compute the gradients of the loss function with respect to each parameter. +The transformation matrix :math:`W` and bias vector :math:`b` are the *parameters* of the layer. The *parameters* of a layer are learned during training in the *backward pass*. The backward pass computes the gradients of the output function with respect to all parameters and inputs. The optimizer can use chain rule to compute the gradients of the loss function with respect to each parameter. Suppose our loss function is :math:`c(y)`, then @@ -37,7 +37,7 @@ Suppose our loss function is :math:`c(y)`, then \frac{\partial c(y)}{\partial x} = \frac{\partial c(y)}{\partial y} \frac{\partial y}{\partial x} -Suppose :math:`z = f(W^T x + b)`, then +Suppose :math:`z = W^T x + b`, then .. math:: @@ -48,7 +48,7 @@ This derivative can be automatically computed by our base layer class. Then, for fully connected layer, we need to compute: .. math:: - + \frac{\partial z}{\partial x} = W, \frac{\partial z_j}{\partial W_{ij}} = x_i, \frac{\partial z}{\partial b} = \mathbf 1 where :math:`\mathbf 1` is an all one vector, :math:`W_{ij}` is the number at the i-th row and j-th column of the matrix :math:`W`, :math:`z_j` is the j-th component of the vector :math:`z`, and :math:`x_i` is the i-th component of the vector :math:`x`. @@ -322,7 +322,7 @@ All the gradient check unit tests are located in :code:`paddle/gserver/tests/tes /* weight */ true); } } - + If you are creating a new file for the test, such as :code:`paddle/gserver/tests/testFCGrad.cpp`, you need to add the file to :code:`paddle/gserver/tests/CMakeLists.txt`. An example is given below. All the unit tests will run when you execute the command :code:`make tests`. Notice that some layers might need high accuracy for the gradient check unit tests to work well. You need to configure :code:`WITH_DOUBLE` to `ON` when configuring cmake. .. code-block:: bash diff --git a/go/CMakeLists.txt b/go/CMakeLists.txt index f00c70a0589a4f41a23164a95d505d4310d9157b..29ce909c6442014fa0b64c6ca018a61b92c840e9 100644 --- a/go/CMakeLists.txt +++ b/go/CMakeLists.txt @@ -17,3 +17,7 @@ add_subdirectory(pserver/client/c) add_subdirectory(cmd/pserver) add_subdirectory(cmd/master) add_subdirectory(master/c) +add_subdirectory(master) +add_subdirectory(pserver) +add_subdirectory(pserver/client) +add_subdirectory(utils/networkhelper) diff --git a/go/cmd/master/CMakeLists.txt b/go/cmd/master/CMakeLists.txt index 1058ffa86b3f00b5e9525edca39a843da9b62db8..9e149967e71c9439bb00b973aa8723a809604aaf 100644 --- a/go/cmd/master/CMakeLists.txt +++ b/go/cmd/master/CMakeLists.txt @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -go_binary(master SRC master.go DEPS paddle_go_optimizer) +go_binary(master SRC master.go) diff --git a/go/cmd/pserver/pserver.go b/go/cmd/pserver/pserver.go index 0ecb1242c3c3d5246125c9ce946001ccf9cbec24..b331b8126cadc2c5df516fb241913415b2e3e73d 100644 --- a/go/cmd/pserver/pserver.go +++ b/go/cmd/pserver/pserver.go @@ -8,6 +8,7 @@ import ( "time" "github.com/namsral/flag" + "github.com/topicai/candy" "github.com/PaddlePaddle/Paddle/go/pserver" log "github.com/sirupsen/logrus" @@ -18,53 +19,47 @@ func main() { index := flag.Int("index", -1, "index of this pserver, should be larger or equal than 0") etcdEndpoint := flag.String("etcd-endpoint", "http://127.0.0.1:2379", "comma separated endpoint string for pserver to connect to etcd") - etcdTimeout := flag.Int("etcd-timeout", 5, "timeout for etcd calls") + etcdTimeout := flag.Duration("etcd-timeout", 5*time.Second, "timeout for etcd calls") numPservers := flag.Int("num-pservers", 1, "total pserver count in a training job") checkpointPath := flag.String("checkpoint-path", "/checkpoints/", "save checkpoint path") - checkpointInterval := flag.Int("checkpoint-interval", 600, "save checkpoint per interval seconds") + checkpointInterval := flag.Duration("checkpoint-interval", 600*time.Second, "save checkpoint per interval seconds") logLevel := flag.String("log-level", "info", "log level, possible values: debug, info, warning, error, fatal, panic") flag.Parse() level, err := log.ParseLevel(*logLevel) - if err != nil { - panic(err) - } + candy.Must(err) + log.SetLevel(level) var idx int + var cp pserver.Checkpoint var e *pserver.EtcdClient if *index >= 0 { idx = *index } else { - timeout := time.Second * time.Duration((*etcdTimeout)) - e = pserver.NewEtcdClient(*etcdEndpoint, *numPservers, timeout) + e = pserver.NewEtcdClient(*etcdEndpoint, *numPservers, *etcdTimeout) idx, err = e.Register() + candy.Must(err) + + cp, err = pserver.NewCheckpointFromFile(*checkpointPath, idx, e) if err != nil { - panic(err) + log.Errorf("Fetch checkpoint failed, %s", err) } } s, err := pserver.NewService(idx, *checkpointInterval, *checkpointPath, e, cp) - if err != nil { - panic(err) - } + candy.Must(err) + err = rpc.Register(s) - if err != nil { - panic(err) - } + candy.Must(err) rpc.HandleHTTP() l, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) - if err != nil { - panic(err) - } + candy.Must(err) log.Infof("start pserver at port %d", *port) err = http.Serve(l, nil) - - if err != nil { - panic(err) - } + candy.Must(err) } diff --git a/go/glide.lock b/go/glide.lock index 190a222338b24b7edac76c72d07df0b2cbd7d9be..f71ae643d68d29846611ec52d0ae7d67e4ced850 100644 --- a/go/glide.lock +++ b/go/glide.lock @@ -1,8 +1,8 @@ -hash: b8f18ce6784bd3fadd9fed0b8443e7b658234ea785ae1f220723ae2c1f652aa7 -updated: 2017-06-27T14:05:48.925262819+08:00 +hash: a8faea3a363468a88917ddeb3b1c9ea36886fb2c622acbad42604fa9cb4d3855 +updated: 2017-07-11T10:04:40.786745417+08:00 imports: - name: github.com/coreos/etcd - version: 61fc123e7a8b14a0a258aa3f5c4159861b1ec2e7 + version: cb2a496c4ddd1c87a9f280e116649b599999ec79 subpackages: - auth/authpb - clientv3 @@ -22,7 +22,9 @@ imports: - name: github.com/PaddlePaddle/recordio version: edfb82af0739c84f241c87390ec5649c7b28c129 - name: github.com/sirupsen/logrus - version: 202f25545ea4cf9b191ff7f846df5d87c9382c2b + version: 7f976d3a76720c4c27af2ba716b85d2e0a7e38b1 +- name: github.com/topicai/candy + version: 1b9030d056fa9f8c4b1f9c91b52fe4b8ab4cd8cc - name: golang.org/x/net version: c8c74377599bd978aee1cf3b9b63a8634051cec2 subpackages: @@ -34,11 +36,11 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: f7928cfef4d09d1b080aa2b6fd3ca9ba1567c733 + version: abf9c25f54453410d0c6668e519582a9e1115027 subpackages: - unix - name: golang.org/x/text - version: 4e9ab9ee170f2a39bd66c92b3e0a47ff47a4bc77 + version: cfdf022e86b4ecfb646e1efbd7db175dd623a8fa subpackages: - secure/bidirule - transform diff --git a/go/glide.yaml b/go/glide.yaml index 05c5d15ca22b6a3d85bee8e1f31d222034ce5314..ab472c7cda9755d0399bb8376b16589be8b53057 100644 --- a/go/glide.yaml +++ b/go/glide.yaml @@ -10,3 +10,4 @@ import: version: ^1.7.4-pre - package: github.com/sirupsen/logrus version: ^1.0.0 +- package: github.com/topicai/candy diff --git a/go/master/CMakeLists.txt b/go/master/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..30531e6469297be1624ea590ea71b1c996b58ed4 --- /dev/null +++ b/go/master/CMakeLists.txt @@ -0,0 +1,3 @@ +if(WITH_TESTING) + go_test(master_test) +endif() diff --git a/go/master/client.go b/go/master/client.go index 05383f1bf40c0e2b1f974e802ee8fd6aac109b00..a2ca3f3ef8ce300e3df09a302d74b56ee23c6d10 100644 --- a/go/master/client.go +++ b/go/master/client.go @@ -68,7 +68,7 @@ func (c *Client) getRecords() { // We treat a task as finished whenever the last data // instance of the task is read. This is not exactly // correct, but a reasonable approximation. - c.taskFinished(t.ID) + c.taskFinished(t.Meta.ID) } } @@ -118,6 +118,11 @@ func (c *Client) taskFinished(taskID int) error { return c.conn.Call("Service.TaskFinished", taskID, nil) } +// TaskFailed tell the master server as task is failed. +func (c *Client) taskFailed(meta TaskMeta) error { + return c.conn.Call("Service.TaskFailed", meta, nil) +} + // NextRecord returns next record in the dataset. // // NextRecord will block until the next record is available. It is diff --git a/go/master/client_internal_test.go b/go/master/client_internal_test.go index 364dce7b58cf6366af711bde9107559a762563a4..49263474c8fe2410ffb6db93a9647f5ab066b06b 100644 --- a/go/master/client_internal_test.go +++ b/go/master/client_internal_test.go @@ -95,10 +95,16 @@ func TestGetFinishTask(t *testing.T) { t.Fatalf("Should get error, pass: %d\n", i) } - err = c.taskFinished(tasks[0].ID) + err = c.taskFinished(tasks[0].Meta.ID) if err != nil { t.Fatalf("Error: %v, pass: %d\n", err, i) } + + err = c.taskFailed(tasks[0].Meta) + if err != nil { + t.Fatalf("Error: %v, pass: %d\n", err, i) + } + tasks = tasks[1:] task, err := c.getTask() if err != nil { @@ -107,7 +113,7 @@ func TestGetFinishTask(t *testing.T) { tasks = append(tasks, task) for _, task := range tasks { - err = c.taskFinished(task.ID) + err = c.taskFinished(task.Meta.ID) if err != nil { t.Fatalf("Error: %v, pass: %d\n", err, i) } diff --git a/go/master/service.go b/go/master/service.go index 58e68e744859933aa74cac231356d4ff9dfb8d7b..a6050ab99437244dade83f2943f6649453d47fff 100644 --- a/go/master/service.go +++ b/go/master/service.go @@ -31,30 +31,36 @@ type Chunk struct { Index recordio.Index // chunk index } +// TaskMeta is a struct which stores task's meta info. +type TaskMeta struct { + ID int + Epoch int +} + // Task is the basic unit of data instances assigned to trainers. type Task struct { - ID int + Meta TaskMeta Chunks []Chunk } type taskEntry struct { - Epoch int - NumTimeout int - Task Task + Task Task + // A task fails if it's timeout or trainer reports it exits unnormally. + NumFailure int } type taskQueues struct { Todo []taskEntry Pending map[int]taskEntry // map from task ID to task entry Done []taskEntry - Failed []Task + Failed []taskEntry } // Service is the master server service. type Service struct { chunksPerTask int timeoutDur time.Duration - timeoutMax int + failureMax int ready chan struct{} store Store @@ -73,7 +79,7 @@ func partition(chunks []Chunk, chunksPerTask int) []taskEntry { var cur taskEntry for i, c := range chunks { if i%chunksPerTask == 0 && len(cur.Task.Chunks) > 0 { - cur.Task.ID = id + cur.Task.Meta.ID = id id++ result = append(result, cur) cur.Task.Chunks = nil @@ -83,7 +89,7 @@ func partition(chunks []Chunk, chunksPerTask int) []taskEntry { } if len(cur.Task.Chunks) > 0 { - cur.Task.ID = id + cur.Task.Meta.ID = id result = append(result, cur) } @@ -91,11 +97,11 @@ func partition(chunks []Chunk, chunksPerTask int) []taskEntry { } // NewService creates a new service. -func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, timeoutMax int) (*Service, error) { +func NewService(store Store, chunksPerTask int, timeoutDur time.Duration, failureMax int) (*Service, error) { s := &Service{} s.chunksPerTask = chunksPerTask s.timeoutDur = timeoutDur - s.timeoutMax = timeoutMax + s.failureMax = failureMax s.taskQueues = taskQueues{} s.taskQueues.Pending = make(map[int]taskEntry) s.ready = make(chan struct{}) @@ -257,6 +263,34 @@ func (s *Service) SetDataset(globPaths []string, dummy *int) error { return nil } +func (s *Service) processFailedTask(t taskEntry, epoch int) { + if t.Task.Meta.Epoch != epoch { + // new epoch, task launched after the + // schedule of this timeout check or failed status report. + return + } + + defer func() { + err := s.snapshot() + if err != nil { + log.Errorln(err) + } + }() + + delete(s.taskQueues.Pending, t.Task.Meta.ID) + + t.NumFailure++ + if t.NumFailure > s.failureMax { + log.Warningf("Task %v failed %d times, discard.", t.Task, t.NumFailure) + s.taskQueues.Failed = append(s.taskQueues.Failed, t) + return + } + + log.Warningf("Task %v failed %d times, discard.", t.Task, t.NumFailure) + s.taskQueues.Todo = append(s.taskQueues.Todo, t) + return +} + func (s *Service) checkTimeoutFunc(taskID int, epoch int) func() { return func() { s.mu.Lock() @@ -267,30 +301,7 @@ func (s *Service) checkTimeoutFunc(taskID int, epoch int) func() { return } - if t.Epoch != epoch { - // new epoch, task launched after the - // schedule of this timeout check. - return - } - - defer func() { - err := s.snapshot() - if err != nil { - log.Errorln(err) - } - }() - - delete(s.taskQueues.Pending, t.Task.ID) - - t.NumTimeout++ - if t.NumTimeout > s.timeoutMax { - log.Warningf("Task %v timed out %d times, discard.", t.Task, t.NumTimeout) - s.taskQueues.Failed = append(s.taskQueues.Failed, t.Task) - return - } - - log.Warningf("Task %v timed out %d times, retry.", t.Task, t.NumTimeout) - s.taskQueues.Todo = append(s.taskQueues.Todo, t) + s.processFailedTask(t, epoch) } } @@ -339,18 +350,18 @@ func (s *Service) GetTask(dummy int, task *Task) error { } t := s.taskQueues.Todo[0] - t.Epoch++ + t.Task.Meta.Epoch++ s.taskQueues.Todo = s.taskQueues.Todo[1:] - s.taskQueues.Pending[t.Task.ID] = t + s.taskQueues.Pending[t.Task.Meta.ID] = t err := s.snapshot() if err != nil { return err } *task = t.Task - log.WithFields(s.logFields()).Infof("Task #%d dispatched.", task.ID) + log.WithFields(s.logFields()).Infof("Task #%v dispatched.", t.Task.Meta) - time.AfterFunc(s.timeoutDur, s.checkTimeoutFunc(t.Task.ID, t.Epoch)) + time.AfterFunc(s.timeoutDur, s.checkTimeoutFunc(t.Task.Meta.ID, t.Task.Meta.Epoch)) return nil } @@ -365,13 +376,12 @@ func (s *Service) TaskFinished(taskID int, dummy *int) error { t, ok := s.taskQueues.Pending[taskID] if !ok { - err := errors.New("pending task not found") log.WithFields(s.logFields()).Warningln("Pending task #%d not found.", taskID) - return err + return nil } // task finished, reset timeout - t.NumTimeout = 0 + t.NumFailure = 0 s.taskQueues.Done = append(s.taskQueues.Done, t) delete(s.taskQueues.Pending, taskID) @@ -389,3 +399,22 @@ func (s *Service) TaskFinished(taskID int, dummy *int) error { } return err } + +// TaskFailed tells the service that a task is failed. +func (s *Service) TaskFailed(meta TaskMeta, dummy *int) error { + select { + case <-s.ready: + } + + s.mu.Lock() + defer s.mu.Unlock() + + t, ok := s.taskQueues.Pending[meta.ID] + if !ok { + log.WithFields(s.logFields()).Warningln("TaskFailed:Pending task #%v not found.", t.Task.Meta) + return nil + } + + s.processFailedTask(t, meta.Epoch) + return nil +} diff --git a/go/master/service_internal_test.go b/go/master/service_internal_test.go index bc435b505c014ca13ed5fc16b33a21ebb089a3b7..9c0d1d0a39fc8cb2b29fd0e3a4ba0c9b255f80fb 100644 --- a/go/master/service_internal_test.go +++ b/go/master/service_internal_test.go @@ -30,7 +30,7 @@ func TestPartionIndex(t *testing.T) { cs := make([]Chunk, 100) ts := partition(cs, 20) for i := range ts { - if ts[i].Task.ID != i { + if ts[i].Task.Meta.ID != i { t.Error(ts[i], i) } } diff --git a/go/pserver/CMakeLists.txt b/go/pserver/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..6267040a6eb421ef5006a83625cf24a8124f5320 --- /dev/null +++ b/go/pserver/CMakeLists.txt @@ -0,0 +1,3 @@ +if(WITH_TESTING) + go_test(pserver_test DEPS paddle_go_optimizer) +endif() diff --git a/go/pserver/client/CMakeLists.txt b/go/pserver/client/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0052bb460bbe3a8fc1e898cac8c3d42caec098a7 --- /dev/null +++ b/go/pserver/client/CMakeLists.txt @@ -0,0 +1,3 @@ +if(WITH_TESTING) + go_test(pserver_client_test DEPS paddle_go_optimizer) +endif() diff --git a/go/pserver/client/c/.gitignore b/go/pserver/client/c/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4bf05c85386dfcef83453a663dffc5d62efcbcc0 --- /dev/null +++ b/go/pserver/client/c/.gitignore @@ -0,0 +1 @@ +libpaddle_go_optimizer.a diff --git a/go/pserver/client/c/CMakeLists.txt b/go/pserver/client/c/CMakeLists.txt index 93a0a27f858f8654e0a6114abe7e326b086b8bf9..c6333eab550c9a2b71bcaf20b69b2bc0a9b9c529 100644 --- a/go/pserver/client/c/CMakeLists.txt +++ b/go/pserver/client/c/CMakeLists.txt @@ -1,5 +1,13 @@ cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf) target_link_libraries(paddle_go_optimizer stdc++ m) + +# Copy library to the required place. +# See: go/pserver/optimizer.go: +# // #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm +add_custom_command(TARGET paddle_go_optimizer POST_BUILD + COMMAND cp "${CMAKE_CURRENT_BINARY_DIR}/libpaddle_go_optimizer.a" "${CMAKE_CURRENT_SOURCE_DIR}" + ) + go_library(paddle_pserver_cclient STATIC DEPS paddle_go_optimizer) if(WITH_TESTING) # FIXME: this test requires pserver which is not managed by the test diff --git a/go/pserver/client/c/test/test_train.py b/go/pserver/client/c/test/test_train.py index 68e1d9b269209b695e27f91a656dc2d8e527b4cd..d6922672f4c1253e62cfe54965f6c2f3b5e6c7bf 100644 --- a/go/pserver/client/c/test/test_train.py +++ b/go/pserver/client/c/test/test_train.py @@ -19,7 +19,7 @@ def main(): # create parameters parameters = paddle.parameters.create(cost) - # create optimizer + # create optimizer of new remote updater to pserver optimizer = paddle.optimizer.Momentum(momentum=0) #TODO(zhihong) : replace optimizer with new OptimizerConfig diff --git a/go/pserver/client/client_test.go b/go/pserver/client/client_test.go index 29b400812c9dc3a5f44700eacbf7ba043248f2f2..27f4ff2380b3b5aa01485838eb4a876a8863d901 100644 --- a/go/pserver/client/client_test.go +++ b/go/pserver/client/client_test.go @@ -3,11 +3,13 @@ package client_test import ( "context" "io/ioutil" + "math/rand" "net" "net/http" "net/rpc" "strconv" "strings" + "sync" "testing" "time" @@ -42,7 +44,8 @@ func initClient() [numPserver]int { ports[i] = p go func(l net.Listener) { - s, err := pserver.NewService(0) + var cp pserver.Checkpoint + s, err := pserver.NewService(0, 1, "", nil, cp) if err != nil { panic(err) } @@ -99,27 +102,34 @@ func (l lister) List() []client.Server { return l } -func ClientTest(t *testing.T, c *client.Client) { +func testClient(t *testing.T, c *client.Client) { selected := c.BeginInitParams() if !selected { t.Fatal("should be selected.") } - const numParameter = 100 + const numParameter = 1000 config, err := ioutil.ReadFile("./c/test/testdata/optimizer.pb") if err != nil { t.Fatalf("read optimizer proto failed") } + + var wg sync.WaitGroup for i := 0; i < numParameter; i++ { - var p pserver.Parameter - p.Name = "p_" + strconv.Itoa(i) - p.ElementType = pserver.Float32 - p.Content = make([]byte, (i+1)*100) - err := c.InitParam(pserver.ParameterWithConfig{Param: p, Config: config}) - if err != nil { - t.Fatal(err) - } + wg.Add(1) + go func(i int) { + var p pserver.Parameter + p.Name = "p_" + strconv.Itoa(i) + p.ElementType = pserver.Float32 + p.Content = make([]byte, (i+1)*100) + err := c.InitParam(pserver.ParameterWithConfig{Param: p, Config: config}) + if err != nil { + t.Fatal(err) + } + wg.Done() + }(i) } + wg.Wait() err = c.FinishInitParams() if err != nil { @@ -127,7 +137,7 @@ func ClientTest(t *testing.T, c *client.Client) { } var grads []pserver.Gradient - for i := 0; i < numParameter/2; i++ { + for i := 0; i < numParameter; i++ { var g pserver.Gradient g.Name = "p_" + strconv.Itoa(i) g.ElementType = pserver.Float32 @@ -135,9 +145,31 @@ func ClientTest(t *testing.T, c *client.Client) { grads = append(grads, g) } - err = c.SendGrads(grads) - if err != nil { - t.Fatal(err) + const paramPerGroup = 10 + const numGroups = numParameter / paramPerGroup + + // shuffle send grads order + for i := range grads { + j := rand.Intn(i + 1) + grads[i], grads[j] = grads[j], grads[i] + } + + for i := 0; i < numGroups; i++ { + var gs []pserver.Gradient + if i == numGroups-1 { + gs = grads[i*paramPerGroup:] + } else { + gs = grads[i*paramPerGroup : (i+1)*paramPerGroup] + } + + wg.Add(1) + go func(gs []pserver.Gradient) { + err = c.SendGrads(gs) + if err != nil { + t.Fatal(err) + } + wg.Done() + }(gs) } names := make([]string, numParameter) @@ -145,20 +177,35 @@ func ClientTest(t *testing.T, c *client.Client) { names[i] = "p_" + strconv.Itoa(i) } - params, err := c.GetParams(names) - if err != nil { - t.Fatal(err) - } + for i := 0; i < numGroups; i++ { + var ns []string + if i == numGroups-1 { + ns = names[i*paramPerGroup:] + } else { + ns = names[i*paramPerGroup : (i+1)*paramPerGroup] + } - if len(names) != len(params) { - t.Fatalf("parameter size not match, need: %d, have: %d", len(names), len(params)) - } + wg.Add(1) + go func(ns []string) { + params, err := c.GetParams(ns) + if err != nil { + t.Fatal(err) + } - for i := range params { - if names[i] != params[i].Name { - t.Fatalf("order of returned parameter does not required: parameter name: %s, required name: %s", names[i], params[i].Name) - } + if len(ns) != len(params) { + t.Fatalf("parameter size not match, need: %d, have: %d", len(names), len(params)) + } + + for i := range params { + if ns[i] != params[i].Name { + t.Fatalf("order of returned parameter does not required: parameter name: %s, required name: %s", ns[i], params[i].Name) + } + } + wg.Done() + }(ns) } + + wg.Wait() } func TestNativeClient(t *testing.T) { @@ -168,13 +215,14 @@ func TestNativeClient(t *testing.T) { servers[i] = client.Server{Index: i, Addr: ":" + strconv.Itoa(pserverClientPorts[i])} } c1 := client.NewClient(lister(servers), len(servers), selector(true)) - ClientTest(t, c1) + testClient(t, c1) } -// TODO: tmperary disable etcdClient test for dependency of etcd) +// EtcdClient is a disabled test, since we have not embedded etcd into +// our test. func EtcdClient(t *testing.T) { initEtcdClient() - etcd_client := client.NewEtcd(etcdEndpoints) - c2 := client.NewClient(etcd_client, etcd_client.Desired(), selector(true)) - ClientTest(t, c2) + etcdClient := client.NewEtcd(etcdEndpoints) + c2 := client.NewClient(etcdClient, etcdClient.Desired(), selector(true)) + testClient(t, c2) } diff --git a/go/pserver/etcd_client.go b/go/pserver/etcd_client.go index 1f77787150d16052e3588e9c1795c8d5dafa08e6..4a694b97f47b2ab85d1e109ef7545d104194b5cf 100644 --- a/go/pserver/etcd_client.go +++ b/go/pserver/etcd_client.go @@ -16,7 +16,7 @@ import ( const ( // PsDesired is etcd path for store desired pserver count PsDesired = "/ps_desired" - // PsAddr is the base dir for pserver to store their addr + // PsPath is the base dir for pserver to store their addr PsPath = "/ps/" // PsCheckpoint is the etcd path for store checkpoints information PsCheckpoint = "/checkpoints/" @@ -189,9 +189,25 @@ func (e *EtcdClient) registerPserverEtcd(ctx context.Context) (int, error) { return idx, nil } +// GetKey gets the value by the specified key +func (e *EtcdClient) GetKey(key string, timeout time.Duration) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + resp, err := e.etcdClient.Get(ctx, key) + cancel() + if err != nil { + return []byte{}, err + } + kvs := resp.Kvs + if len(kvs) == 0 { + return []byte{}, nil + } + v := kvs[0].Value + return v, nil +} + // PutKey put into etcd with value by key specified -func (e *EtcdClient) PutKey(key string, value []byte, timeout int) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout)) +func (e *EtcdClient) PutKey(key string, value []byte, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) _, err := e.etcdClient.Put(ctx, key, string(value)) cancel() if err != nil { diff --git a/go/pserver/optimizer.go b/go/pserver/optimizer.go index 2d7882d1a75ef55df4a1ec81a8606cd84334fa64..d6b7fafd59c0453b9f40019166d01ebdc9775117 100644 --- a/go/pserver/optimizer.go +++ b/go/pserver/optimizer.go @@ -1,8 +1,7 @@ package pserver // #cgo CFLAGS: -I ../../ -// //FIXME: ldflags contain "build" path -// #cgo LDFLAGS: ${SRCDIR}/../../build/go/pserver/client/c/libpaddle_go_optimizer.a -lstdc++ -lm +// #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm // #include "paddle/optimizer/optimizer.h" // #include // #include @@ -20,6 +19,7 @@ var nullPtr = unsafe.Pointer(uintptr(0)) type optimizer struct { opt *C.struct_paddle_optimizer elementType ElementType + contentLen int } func cArrayToSlice(p unsafe.Pointer, len int) []byte { @@ -38,25 +38,28 @@ func cArrayToSlice(p unsafe.Pointer, len int) []byte { func newOptimizer(paramWithConfigs ParameterWithConfig, State []byte) *optimizer { o := &optimizer{} o.elementType = paramWithConfigs.Param.ElementType + o.contentLen = len(paramWithConfigs.Param.Content) p := paramWithConfigs.Param c := paramWithConfigs.Config s := State + paramBufferSize := C.size_t(len(p.Content)) log.WithFields(log.Fields{ "ElementType": p.ElementType, - "ParamSize": len(p.Content), + "ParamSize": paramBufferSize, "ConfigSize": len(c), "StateSize": len(s), }).Info("New Optimizer Created with config:") var cbuffer unsafe.Pointer - cbuffer = C.malloc(C.size_t(len(p.Content))) - C.memcpy(cbuffer, unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content))) + cbuffer = C.malloc(paramBufferSize) + + C.memcpy(cbuffer, unsafe.Pointer(&p.Content[0]), paramBufferSize) var cstate unsafe.Pointer if len(s) != 0 { cstate = unsafe.Pointer(&s[0]) } o.opt = C.paddle_create_optimizer((*C.uchar)(&c[0]), C.int(len(c)), - C.paddle_element_type(p.ElementType), cbuffer, C.int(len(p.Content)/C.sizeof_float), (*C.char)(cstate), C.int(len(s))) + C.paddle_element_type(p.ElementType), cbuffer, C.int(paramBufferSize), (*C.char)(cstate), C.int(len(s))) return o } @@ -68,8 +71,8 @@ func (o *optimizer) GetWeights() []byte { func (o *optimizer) GetStates() []byte { var cbuffer *C.char - cbuffer_len := C.paddle_optimizer_get_state(o.opt, &cbuffer) - return cArrayToSlice(unsafe.Pointer(cbuffer), int(cbuffer_len)) + cbufferLen := C.paddle_optimizer_get_state(o.opt, &cbuffer) + return cArrayToSlice(unsafe.Pointer(cbuffer), int(cbufferLen)) } func (o *optimizer) UpdateParameter(g Gradient) error { @@ -77,7 +80,11 @@ func (o *optimizer) UpdateParameter(g Gradient) error { return fmt.Errorf("Name: %s, parameter and gradient element type not match, parameter: %v, gradient: %v", g.Name, o.elementType, g.ElementType) } - r := C.paddle_update_parameter(o.opt, C.paddle_element_type(g.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content))/C.sizeof_float) + if o.contentLen != len(g.Content) { + return fmt.Errorf("Name: %s, parameter and gradient does not have same content len, parameter: %d, gradient: %d", g.Name, o.contentLen, len(g.Content)) + } + + r := C.paddle_update_parameter(o.opt, C.paddle_element_type(g.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content))) if r != 0 { return fmt.Errorf("optimizer update returned error code: %d", r) } diff --git a/go/pserver/service.go b/go/pserver/service.go index 6b52d0d896f8bc04fab6c9b68911523cbb7ac8b9..fec2ec61dc67756439d9fa478788d1f155876538 100644 --- a/go/pserver/service.go +++ b/go/pserver/service.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "strconv" @@ -21,14 +22,14 @@ import ( // ElementType is the type of elements of a Parameter. type ElementType int +// RPC error message. const ( - // AlreadyInitialized is true if pserver is initialized - AlreadyInitialized = "pserver already initialized" - // Uninitialized is true if pserver not fully initialized - Uninitialized = "pserver not fully initialized" + AlreadyInitialized = "pserver already initialized" + Uninitialized = "pserver not fully initialized" + CheckpointMD5Failed = "checkpoint file MD5 validation failed" ) -// Supported element types +// Supported element types. const ( Int32 ElementType = iota UInt32 @@ -51,21 +52,15 @@ type ParameterWithConfig struct { Config []byte // parameter configuration in Proto Buffer format } -// ParameterCheckpoint is Parameter and State checkpoint -type ParameterCheckpoint struct { - ParamConfig ParameterWithConfig - State []byte -} - -// checkpoint signature +// checkpointMeta saves checkpoint metadata type checkpointMeta struct { UUID string `json:"uuid"` - Md5sum string `json:"md5sum"` - Timestamp string `json:"timestamp"` + MD5 string `json:"md5"` + Timestamp int64 `json:"timestamp"` } // Checkpoint is the pserver shard persist in file -type Checkpoint []ParameterCheckpoint +type Checkpoint []parameterCheckpoint // Gradient is the gradient of the parameter. type Gradient Parameter @@ -81,12 +76,53 @@ type Service struct { optMap map[string]*optimizer } +// parameterCheckpoint saves parameter checkpoint +type parameterCheckpoint struct { + ParameterWithConfig + State []byte +} + +// NewCheckpointFromFile loads parameters and state from checkpoint file +func NewCheckpointFromFile(cpPath string, idx int, e *EtcdClient) (Checkpoint, error) { + v, err := e.GetKey(PsPath+string(idx), 3*time.Second) + if err != nil { + return nil, err + } + + var cpMeta checkpointMeta + if err = json.Unmarshal(v, &cpMeta); err != nil { + return nil, err + } + + fn := filepath.Join(cpPath, cpMeta.UUID) + if _, err = os.Stat(fn); os.IsNotExist(err) { + return nil, err + } + content, err := ioutil.ReadFile(fn) + if err != nil { + return nil, err + } + + h := md5.New() + md5 := hex.EncodeToString(h.Sum(content)) + if md5 != cpMeta.MD5 { + return nil, errors.New(CheckpointMD5Failed) + } + + dec := gob.NewDecoder(bytes.NewReader(content)) + cp := Checkpoint{} + if err = dec.Decode(cp); err != nil { + return nil, err + } + return cp, nil +} + // NewService creates a new service, will bypass etcd registration if no -// endpoints specified. -func NewService(idx int, seconds int, path string, client *EtcdClient, cp Checkpoint) (*Service, error) { +// endpoints specified. It will recovery from checkpoint file if a exists a specified checkpoint. +func NewService(idx int, interval time.Duration, path string, client *EtcdClient, cp Checkpoint) (*Service, error) { s := &Service{ idx: idx, - checkpointInterval: time.Second * time.Duration(seconds), + checkpointInterval: interval, checkpointPath: path, client: client, } @@ -95,9 +131,11 @@ func NewService(idx int, seconds int, path string, client *EtcdClient, cp Checkp if cp != nil { for _, item := range cp { - p := item.ParamConfig - st := item.State - s.optMap[p.Param.Name] = newOptimizer(p, st) + p := ParameterWithConfig{ + Param: item.Param, + Config: item.Config, + } + s.optMap[p.Param.Name] = newOptimizer(p, item.State) } } return s, nil @@ -186,13 +224,13 @@ func (s *Service) doCheckpoint() error { s.mu.Lock() defer s.mu.Unlock() - cp := make([]ParameterCheckpoint, 0, len(s.optMap)) + cp := make([]parameterCheckpoint, len(s.optMap)) index := 0 for name, opt := range s.optMap { - var pc ParameterCheckpoint - pc.ParamConfig.Param.Name = name - pc.ParamConfig.Param.ElementType = opt.elementType - pc.ParamConfig.Param.Content = opt.GetWeights() + var pc parameterCheckpoint + pc.Param.Name = name + pc.Param.ElementType = opt.elementType + pc.Param.Content = opt.GetWeights() pc.State = opt.GetStates() cp[index] = pc index++ @@ -206,12 +244,12 @@ func (s *Service) doCheckpoint() error { cpMeta := checkpointMeta{} cpMeta.UUID = s.checkpointPath + strconv.Itoa(s.idx) - cpMeta.Timestamp = time.Now().String() + cpMeta.Timestamp = time.Now().UnixNano() h := md5.New() - cpMeta.Md5sum = hex.EncodeToString(h.Sum(buf.Bytes())) + cpMeta.MD5 = hex.EncodeToString(h.Sum(buf.Bytes())) cpMetajson, _ := json.Marshal(cpMeta) - err = s.client.PutKey(filepath.Join(PsCheckpoint, strconv.Itoa(s.idx)), cpMetajson, 3) + err = s.client.PutKey(filepath.Join(PsCheckpoint, strconv.Itoa(s.idx)), cpMetajson, 3*time.Second) if err != nil { return err } @@ -219,7 +257,11 @@ func (s *Service) doCheckpoint() error { log.Info("checkpoint does not exists.") } else { err = os.Remove(cpMeta.UUID) - log.Infof("checkpoint %s already exsits, removing ", cpMeta.UUID) + if err != nil { + log.Infof("Removing checkpoint %s failed", cpMeta.UUID) + } else { + log.Infof("checkpoint %s already exsits, removing ", cpMeta.UUID) + } } f, err := os.Create(cpMeta.UUID) defer f.Close() diff --git a/go/pserver/service_test.go b/go/pserver/service_test.go index 9bf1a48a596f3e3e73a2e4df651855fd5f4e775f..a191f689fea9b5e64204c3ddfd12edf92f5ddb09 100644 --- a/go/pserver/service_test.go +++ b/go/pserver/service_test.go @@ -31,7 +31,7 @@ func TestServiceFull(t *testing.T) { err = s.InitParam(pserver.ParameterWithConfig{Param: p, Config: config}, nil) if err != nil { - t.FailNow() + t.Fatal(err) } var p1 pserver.Parameter @@ -40,40 +40,40 @@ func TestServiceFull(t *testing.T) { p1.ElementType = pserver.Float32 err = s.InitParam(pserver.ParameterWithConfig{Param: p1, Config: config}, nil) if err != nil { - t.FailNow() + t.Fatal(err) } err = s.FinishInitParams(0, nil) if err != nil { - t.FailNow() + t.Fatal(err) } var param pserver.Parameter err = s.GetParam("param_b", ¶m) if err != nil { - t.FailNow() + t.Fatal(err) } if !reflect.DeepEqual(param, p1) { - t.FailNow() + t.Fatal("not equal:", param, p1) } g1, g2 := pserver.Gradient(p1), pserver.Gradient(p) err = s.SendGrad(g1, nil) if err != nil { - t.FailNow() + t.Fatal(err) } err = s.SendGrad(g2, nil) if err != nil { - t.FailNow() + t.Fatal(err) } var param1 pserver.Parameter err = s.GetParam("param_a", ¶m1) if err != nil { - t.FailNow() + t.Fatal(err) } // don't compare content, since it's already changed by @@ -82,7 +82,7 @@ func TestServiceFull(t *testing.T) { p.Content = nil if !reflect.DeepEqual(param1, p) { - t.FailNow() + t.Fatal("not equal:", param1, p) } } @@ -90,16 +90,16 @@ func TestMultipleInit(t *testing.T) { var cp pserver.Checkpoint s, err := pserver.NewService(0, 1, "", nil, cp) if err != nil { - t.Error(err) + t.Fatal(err) } err = s.FinishInitParams(0, nil) if err != nil { - t.FailNow() + t.Fatal(err) } err = s.FinishInitParams(0, nil) if err.Error() != pserver.AlreadyInitialized { - t.FailNow() + t.Fatal(err) } } @@ -108,7 +108,7 @@ func TestUninitialized(t *testing.T) { s, err := pserver.NewService(0, 1, "", nil, cp) err = s.SendGrad(pserver.Gradient{}, nil) if err.Error() != pserver.Uninitialized { - t.FailNow() + t.Fatal(err) } } @@ -154,12 +154,12 @@ func TestBlockUntilInitialized(t *testing.T) { err = s.InitParam(pserver.ParameterWithConfig{Param: p, Config: config}, nil) if err != nil { - t.FailNow() + t.Fatal(err) } err = s.FinishInitParams(0, nil) if err != nil { - t.FailNow() + t.Fatal(err) } wg.Wait() diff --git a/go/utils/networkhelper/CMakeLists.txt b/go/utils/networkhelper/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..db6cf211d8c0b124856ca5c5fd2c49763b1b4a64 --- /dev/null +++ b/go/utils/networkhelper/CMakeLists.txt @@ -0,0 +1,3 @@ +if(WITH_TESTING) + go_test(network_helper_test) +endif() diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index 307e99bbe3a833f1fe26057ec38d0b96e04bc0fe..4b06966fba2bc9f92756be0cb8110bbcd5272423 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -8,13 +8,14 @@ add_subdirectory(gserver) add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) -add_subdirectory(optimizer) add_subdirectory(string) if(Boost_FOUND) add_subdirectory(memory) add_subdirectory(platform) add_subdirectory(framework) + add_subdirectory(operators) + add_subdirectory(pybind) endif() if(WITH_C_API) diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index aecc97d4a86886ef18036f6eaf5bac0590409777..8415ce67e90397b31864a90cda54b81c19b3b34e 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -11,7 +11,10 @@ proto_library(op_proto SRCS op_proto.proto DEPS attr_type) cc_test(op_proto_test SRCS op_proto_test.cc DEPS op_proto protobuf) proto_library(op_desc SRCS op_desc.proto DEPS attr_type) cc_test(op_desc_test SRCS op_desc_test.cc DEPS op_desc protobuf) -cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_proto op_desc) +cc_library(operator SRCS operator.cc DEPS op_desc device_context) +cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) +cc_library(op_registry SRCS op_registry.cc DEPS op_proto op_desc) +cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry operator) py_proto_compile(framework_py_proto SRCS attr_type.proto op_proto.proto op_desc.proto) # Generate an empty __init__.py to make framework_py_proto as a valid python module. add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) diff --git a/paddle/framework/dim.h b/paddle/framework/dim.h index bcde291d12d429a3f2cd41fa6d0ee606c7c9c92f..883fdc55eb929ebc51e8ae05938e9d07374406ce 100644 --- a/paddle/framework/dim.h +++ b/paddle/framework/dim.h @@ -266,29 +266,6 @@ HOSTDEVICE inline bool contained(const Dim<1>& idx, const Dim<1>& size) { return ((0 <= idx.head) && (idx.head < size.head)); } -/** - * \brief Check if a size and a stride create a Fortran order contiguous - * block of memory. - */ -template -HOST bool contiguous(const Dim& size, const Dim& stride, int mul = 1) { - if (product(size) == 0) return true; - int contiguous_stride = get<0>(size) == 1 ? 0 : mul; - return (get<0>(stride) == contiguous_stride && - contiguous(size.tail, stride.tail, mul * get<0>(size))); -} - -///\cond HIDDEN -// Base case of contiguous, check the nth stride is the size of -// the prefix multiply of n-1 dims. -template <> -inline bool contiguous(const Dim<1>& size, const Dim<1>& stride, int mul) { - if (get<0>(size) == 0) return true; - int contiguous_stride = get<0>(size) == 1 ? 0 : mul; - return get<0>(stride) == contiguous_stride; -} -///\endcond - /** * \brief Compute exclusive prefix-multiply of a Dim. */ @@ -306,31 +283,6 @@ HOSTDEVICE inline Dim<1> ex_prefix_mul(const Dim<1>& src, int mul) { } ///\endcond -/** - * \brief Calculate strides of a contiguous array of the given size - * - * Sets the stride for any dimension with an extent of 1 to 0. - * \param size Dim object containing the size of the array. - * \param base The base stride to use. - * \return Dim object the same size as \p size with the strides. - */ -template -HOSTDEVICE Dim contiguous_strides(const Dim& size, int base = 1) { - int stride = size.head == 1 ? 0 : base; - return Dim(stride, contiguous_strides(size.tail, base * size.head)); -} - -///\cond HIDDEN - -// Base case of contiguous_strides -template <> -HOSTDEVICE inline Dim<1> contiguous_strides(const Dim<1>& size, int base) { - int stride = size.head == 1 ? 0 : base; - return Dim<1>(stride); -} - -///\endcond - /** * Add two dimensions together */ diff --git a/paddle/framework/dim_test.cu b/paddle/framework/dim_test.cu index 809bf04826637195425a32c054c94e00ef940df9..05217415196f3ec3ce9b5de7cb2f82c9de960ba7 100644 --- a/paddle/framework/dim_test.cu +++ b/paddle/framework/dim_test.cu @@ -58,24 +58,6 @@ TEST(Dim, Equality) { EXPECT_EQ(paddle::framework::get<1>(c), 3); EXPECT_EQ(paddle::framework::get<2>(c), 12); - // contiguous_strides - c = paddle::framework::contiguous_strides(paddle::framework::Dim<3>(10, 1, 10)); - EXPECT_EQ(paddle::framework::get<0>(c), 1); - EXPECT_EQ(paddle::framework::get<1>(c), 0); - EXPECT_EQ(paddle::framework::get<2>(c), 10); - c = paddle::framework::contiguous_strides(paddle::framework::Dim<3>(10, 10, 1)); - EXPECT_EQ(paddle::framework::get<0>(c), 1); - EXPECT_EQ(paddle::framework::get<1>(c), 10); - EXPECT_EQ(paddle::framework::get<2>(c), 0); - c = paddle::framework::contiguous_strides(paddle::framework::Dim<3>(1, 10, 10)); - EXPECT_EQ(paddle::framework::get<0>(c), 0); - EXPECT_EQ(paddle::framework::get<1>(c), 1); - EXPECT_EQ(paddle::framework::get<2>(c), 10); - c = paddle::framework::contiguous_strides(paddle::framework::Dim<3>(2, 3, 4)); - EXPECT_EQ(paddle::framework::get<0>(c), 1); - EXPECT_EQ(paddle::framework::get<1>(c), 2); - EXPECT_EQ(paddle::framework::get<2>(c), 6); - // generate from an index auto size = paddle::framework::make_dim(4, 5, 2); c = paddle::framework::Dim<3>(14, size); @@ -101,16 +83,6 @@ TEST(Dim, Bool) { EXPECT_TRUE(a == a); EXPECT_FALSE(a == b); EXPECT_TRUE(a == c); - - // contiguous check - int x = 4, y = 5, z = 2; - paddle::framework::Dim<3> sizef(x, y, z); - paddle::framework::Dim<3> stridea(1, x, x*y); - paddle::framework::Dim<3> strideb(2, 2*x, 2*x*y); - paddle::framework::Dim<3> stridec(1, x, 2*x*y); - EXPECT_TRUE(paddle::framework::contiguous(sizef, stridea)); - EXPECT_FALSE(paddle::framework::contiguous(sizef, strideb)); - EXPECT_FALSE(paddle::framework::contiguous(sizef, stridec)); } TEST(Dim, Print) { diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc new file mode 100644 index 0000000000000000000000000000000000000000..4b35e04e681b414c36cf6d9aee9e64dd68ba5da9 --- /dev/null +++ b/paddle/framework/op_registry.cc @@ -0,0 +1,36 @@ +#include + +namespace paddle { +namespace framework { + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::INT); +} + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::FLOAT); +} + +template <> +void AttrTypeHelper::SetAttrType(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::STRING); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::INTS); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::FLOATS); +} + +template <> +void AttrTypeHelper::SetAttrType>(AttrProto* attr) { + attr->set_type(paddle::framework::AttrType::STRINGS); +} +} // namespace framework +} // namespace paddle \ No newline at end of file diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 81241b5342d8900c205dd62f2a62dc2496010560..61dfcb704964cd730a8fc9ab6ad394cd47cb4666 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -1,26 +1,15 @@ #pragma once +#include +#include #include "paddle/framework/attr_checker.h" - -//#include "paddle/framework/op_base.h" #include "paddle/framework/op_desc.pb.h" #include "paddle/framework/op_proto.pb.h" +#include "paddle/framework/operator.h" namespace paddle { namespace framework { -//==================For test================// -class OpBase { - public: - std::vector inputs_; - std::vector outputs_; - AttributeMap attr_map_; - - virtual std::string Run() const = 0; - virtual ~OpBase() {} -}; -//=========================================// - // helper class to set attribute type struct AttrTypeHelper { template @@ -64,36 +53,6 @@ struct AttrTypeHelper { } }; -template <> -void AttrTypeHelper::SetAttrType(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::INT); -} - -template <> -void AttrTypeHelper::SetAttrType(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::FLOAT); -} - -template <> -void AttrTypeHelper::SetAttrType(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::STRING); -} - -template <> -void AttrTypeHelper::SetAttrType>(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::INTS); -} - -template <> -void AttrTypeHelper::SetAttrType>(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::FLOATS); -} - -template <> -void AttrTypeHelper::SetAttrType>(AttrProto* attr) { - attr->set_type(paddle::framework::AttrType::STRINGS); -} - // this class not only make proto but also init attribute checkers. class OpProtoAndCheckerMaker { public: @@ -103,28 +62,26 @@ class OpProtoAndCheckerMaker { protected: void AddInput(const std::string& name, const std::string& comment) { auto input = proto_->mutable_inputs()->Add(); - *(input->mutable_name()) = name; - *(input->mutable_comment()) = comment; + *input->mutable_name() = name; + *input->mutable_comment() = comment; } void AddOutput(const std::string& name, const std::string& comment) { auto output = proto_->mutable_outputs()->Add(); - *(output->mutable_name()) = name; - *(output->mutable_comment()) = comment; + *output->mutable_name() = name; + *output->mutable_comment() = comment; } template TypedAttrChecker& AddAttr(const std::string& name, const std::string& comment) { auto attr = proto_->mutable_attrs()->Add(); - *(attr->mutable_name()) = name; - *(attr->mutable_comment()) = comment; + *attr->mutable_name() = name; + *attr->mutable_comment() = comment; AttrTypeHelper::SetAttrType(attr); return op_checker_->AddAttrChecker(name); } - void AddType(const std::string& op_type) { proto_->set_type(op_type); } - void AddComment(const std::string& comment) { *(proto_->mutable_comment()) = comment; } @@ -134,120 +91,127 @@ class OpProtoAndCheckerMaker { }; class OpRegistry { - typedef std::function OpCreator; + using OpCreator = std::function; public: template static void RegisterOp(const std::string& op_type) { - creators_[op_type] = []() { return new OpType; }; - OpProto& op_proto = protos_[op_type]; - OpAttrChecker& op_checker = op_checkers_[op_type]; + creators()[op_type] = [] { return new OpType; }; + OpProto& op_proto = protos()[op_type]; + OpAttrChecker& op_checker = op_checkers()[op_type]; ProtoMakerType(&op_proto, &op_checker); - PADDLE_ENFORCE(op_proto.IsInitialized() == true, - "Fail to initialize %s's OpProto !", op_type); + *op_proto.mutable_type() = op_type; + PADDLE_ENFORCE( + op_proto.IsInitialized(), + "Fail to initialize %s's OpProto, because %s is not initialized", + op_type, op_proto.InitializationErrorString()); } - static OpBase* CreateOp(const OpDesc& op_desc) { + static OperatorBase* CreateOp(const OpDesc& op_desc) { std::string op_type = op_desc.type(); - OpBase* op = (creators_.at(op_type))(); - (op->inputs_).resize(op_desc.inputs_size()); - for (int i = 0; i < op_desc.inputs_size(); ++i) { - (op->inputs_)[i] = op_desc.inputs(i); - } - (op->outputs_).resize(op_desc.outputs_size()); - for (int i = 0; i < op_desc.outputs_size(); ++i) { - (op->outputs_)[i] = op_desc.outputs(i); + OperatorBase* op = creators().at(op_type)(); + op->desc_ = op_desc; + op->inputs_.reserve((size_t)op_desc.inputs_size()); + std::copy(op_desc.inputs().begin(), op_desc.inputs().end(), + std::back_inserter(op->inputs_)); + op->outputs_.reserve((size_t)op_desc.outputs_size()); + std::copy(op_desc.outputs().begin(), op_desc.outputs().end(), + std::back_inserter(op->outputs_)); + for (auto& attr : op_desc.attrs()) { + op->attrs_[attr.name()] = AttrTypeHelper::GetAttrValue(attr); } - for (int i = 0; i < op_desc.attrs_size(); ++i) { - const AttrDesc& ith_attr = op_desc.attrs(i); - std::string name = ith_attr.name(); - (op->attr_map_)[name] = AttrTypeHelper::GetAttrValue(ith_attr); - } - const OpAttrChecker& op_checker = op_checkers_.at(op_type); - op_checker.Check(op->attr_map_); + op_checkers().at(op_type).Check(op->attrs_); + op->Init(); return op; } private: - static std::unordered_map creators_; - static std::unordered_map protos_; - static std::unordered_map op_checkers_; -}; - -std::unordered_map> OpRegistry::creators_; -std::unordered_map OpRegistry::protos_; -std::unordered_map OpRegistry::op_checkers_; - -template -class OpRegisterHelper { - public: - OpRegisterHelper(std::string op_type) { - OpRegistry::RegisterOp(op_type); - } -}; - -#define REGISTER_OP(__op_class, __op_maker_class, __op_type) \ - class __op_class##Register { \ - private: \ - const static OpRegisterHelper<__op_class, __op_maker_class> reg; \ - }; \ - const OpRegisterHelper<__op_class, __op_maker_class> \ - __op_class##Register::reg(#__op_type); - -// Demos - -class CosineOp : public OpBase { - public: - virtual std::string Run() const { - std::string msg = "CosineOp runs! scale = " + - std::to_string(boost::get(attr_map_.at("scale"))); - return msg; - } -}; - -class CosineOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { - public: - CosineOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("input", "input of cosine op"); - AddOutput("output", "output of cosine op"); - AddAttr("scale", "scale of cosine op") - .SetDefault(1.0) - .LargerThan(0.0); - AddType("cos"); - AddComment("This is cos op"); + static std::unordered_map& creators() { + static std::unordered_map creators_; + return creators_; } -}; -REGISTER_OP(CosineOp, CosineOpProtoAndCheckerMaker, cos_sim) + static std::unordered_map& protos() { + static std::unordered_map protos_; + return protos_; + }; -class MyTestOp : public OpBase { - public: - virtual std::string Run() const { - std::string msg = - "MyTestOp runs! test_attr = " + - std::to_string(boost::get(attr_map_.at("test_attr"))); - return msg; - } + static std::unordered_map& op_checkers() { + static std::unordered_map op_checkers_; + return op_checkers_; + }; }; -class MyTestOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { +template +class OpRegisterHelper { public: - MyTestOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("input", "input of cosine op"); - AddOutput("output", "output of cosine op"); - auto my_checker = [](int i) { - PADDLE_ENFORCE(i % 2 == 0, "'test_attr' must be even!"); - }; - AddAttr("test_attr", "a simple test attribute") - .AddCustomChecker(my_checker); - AddType("my_test_op"); - AddComment("This is my_test op"); + OpRegisterHelper(const char* op_type) { + OpRegistry::RegisterOp(op_type); } }; -REGISTER_OP(MyTestOp, MyTestOpProtoAndCheckerMaker, my_test_op) +#define STATIC_ASSERT_GLOBAL_NAMESPACE(uniq_name, msg) \ + struct __test_global_namespace_##uniq_name##__ {}; \ + static_assert(std::is_same<::__test_global_namespace_##uniq_name##__, \ + __test_global_namespace_##uniq_name##__>::value, \ + msg) + +#define REGISTER_OP(__op_type, __op_class, __op_maker_class) \ + STATIC_ASSERT_GLOBAL_NAMESPACE(__reg_op__##__op_type, \ + "REGISTER_OP must be in global namespace"); \ + static ::paddle::framework::OpRegisterHelper<__op_class, __op_maker_class> \ + __op_register_##__op_type##__(#__op_type); \ + int __op_register_##__op_type##_handle__() { return 0; } + +#define REGISTER_OP_KERNEL(type, GPU_OR_CPU, PlaceType, KernelType) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __reg_op_kernel_##type##_##GPU_OR_CPU##__, \ + "REGISTER_OP_KERNEL must be in global namespace"); \ + struct __op_kernel_register__##type##__ { \ + __op_kernel_register__##type##__() { \ + ::paddle::framework::OperatorWithKernel::OpKernelKey key; \ + key.place_ = PlaceType(); \ + ::paddle::framework::OperatorWithKernel::AllOpKernels()[#type][key] \ + .reset(new KernelType()); \ + } \ + }; \ + static __op_kernel_register__##type##__ __reg_kernel_##type##__; \ + int __op_kernel_register_##type##_handle_##GPU_OR_CPU##__() { return 0; } + +#define REGISTER_OP_GPU_KERNEL(type, KernelType) \ + REGISTER_OP_KERNEL(type, GPU, ::paddle::platform::GPUPlace, KernelType) + +#define REGISTER_OP_CPU_KERNEL(type, KernelType) \ + REGISTER_OP_KERNEL(type, CPU, ::paddle::platform::CPUPlace, KernelType) + +#define USE_OP_WITHOUT_KERNEL(op_type) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __use_op_without_kernel_##op_type, \ + "USE_OP_WITHOUT_KERNEL must be in global namespace"); \ + extern int __op_register_##op_type##_handle__(); \ + static int __use_op_ptr_##op_type##_without_kernel__ \ + __attribute__((unused)) = __op_register_##op_type##_handle__() + +#define USE_OP_KERNEL(op_type, DEVICE_TYPE) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __use_op_kernel_##op_type##_##DEVICE_TYPE##__, \ + "USE_OP_KERNEL must be in global namespace"); \ + extern int __op_kernel_register_##op_type##_handle_##DEVICE_TYPE##__(); \ + static int __use_op_ptr_##op_type##_##DEVICE_TYPE##_kernel__ \ + __attribute__((unused)) = \ + __op_kernel_register_##op_type##_handle_##DEVICE_TYPE##__() + +#ifdef PADDLE_ONLY_CPU +#define USE_OP(op_type) \ + USE_OP_WITHOUT_KERNEL(op_type); \ + USE_OP_KERNEL(op_type, CPU); + +#else +#define USE_OP(op_type) \ + USE_OP_WITHOUT_KERNEL(op_type); \ + USE_OP_KERNEL(op_type, CPU); \ + USE_OP_KERNEL(op_type, GPU) +#endif } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index ae6b7387129c6f82d0c63270d5368da65abd3b51..9bcc0407addca555e0b47b8178f6304396ce81fc 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -1,25 +1,78 @@ #include "paddle/framework/op_registry.h" #include +namespace paddle { +namespace framework { +class CosineOp : public OperatorBase { + public: + void Run(const std::shared_ptr& scope, + const platform::DeviceContext& dev_ctx) const override {} + void InferShape(const std::shared_ptr& scope) const override {} +}; + +class CosineOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { + public: + CosineOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of cosine op"); + AddOutput("output", "output of cosine op"); + AddAttr("scale", "scale of cosine op") + .SetDefault(1.0) + .LargerThan(0.0); + AddComment("This is cos op"); + } +}; + +class MyTestOp : public OperatorBase { + public: + void InferShape(const std::shared_ptr& scope) const override {} + void Run(const std::shared_ptr& scope, + const platform::DeviceContext& dev_ctx) const override {} + + public: +}; + +class MyTestOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { + public: + MyTestOpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of cosine op"); + AddOutput("output", "output of cosine op"); + auto my_checker = [](int i) { + PADDLE_ENFORCE(i % 2 == 0, "'test_attr' must be even!"); + }; + AddAttr("test_attr", "a simple test attribute") + .AddCustomChecker(my_checker); + AddComment("This is my_test op"); + } +}; +} // namespace framework +} // namespace paddle + +REGISTER_OP(cos_sim, paddle::framework::CosineOp, + paddle::framework::CosineOpProtoAndCheckerMaker); +REGISTER_OP(my_test_op, paddle::framework::MyTestOp, + paddle::framework::MyTestOpProtoAndCheckerMaker); + TEST(OpRegistry, CreateOp) { paddle::framework::OpDesc op_desc; op_desc.set_type("cos_sim"); op_desc.add_inputs("aa"); op_desc.add_outputs("bb"); + float scale = 3.3; auto attr = op_desc.mutable_attrs()->Add(); attr->set_name("scale"); attr->set_type(paddle::framework::AttrType::FLOAT); - attr->set_f(3.3); + attr->set_f(scale); - paddle::framework::OpBase* op = + paddle::framework::OperatorBase* op = paddle::framework::OpRegistry::CreateOp(op_desc); - std::string debug_str = op->Run(); - std::string str = "CosineOp runs! scale = " + std::to_string(3.3); - ASSERT_EQ(str.size(), debug_str.size()); - for (size_t i = 0; i < debug_str.length(); ++i) { - ASSERT_EQ(debug_str[i], str[i]); - } + auto scope = std::make_shared(); + paddle::platform::CPUDeviceContext dev_ctx; + op->Run(scope, dev_ctx); + float scale_get = op->GetAttr("scale"); + ASSERT_EQ(scale_get, scale); } TEST(OpRegistry, IllegalAttr) { @@ -35,7 +88,7 @@ TEST(OpRegistry, IllegalAttr) { bool caught = false; try { - paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OperatorBase* op __attribute__((unused)) = paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::framework::EnforceNotMet err) { caught = true; @@ -54,15 +107,14 @@ TEST(OpRegistry, DefaultValue) { op_desc.add_inputs("aa"); op_desc.add_outputs("bb"); - paddle::framework::OpBase* op = + ASSERT_TRUE(op_desc.IsInitialized()); + + paddle::framework::OperatorBase* op = paddle::framework::OpRegistry::CreateOp(op_desc); - std::string debug_str = op->Run(); - float default_value = 1.0; - std::string str = "CosineOp runs! scale = " + std::to_string(default_value); - ASSERT_EQ(str.size(), debug_str.size()); - for (size_t i = 0; i < debug_str.length(); ++i) { - ASSERT_EQ(debug_str[i], str[i]); - } + auto scope = std::make_shared(); + paddle::platform::CPUDeviceContext dev_ctx; + op->Run(scope, dev_ctx); + ASSERT_EQ(op->GetAttr("scale"), 1.0); } TEST(OpRegistry, CustomChecker) { @@ -74,7 +126,7 @@ TEST(OpRegistry, CustomChecker) { // attr 'test_attr' is not set bool caught = false; try { - paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OperatorBase* op __attribute__((unused)) = paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::framework::EnforceNotMet err) { caught = true; @@ -93,7 +145,7 @@ TEST(OpRegistry, CustomChecker) { attr->set_i(3); caught = false; try { - paddle::framework::OpBase* op __attribute__((unused)) = + paddle::framework::OperatorBase* op __attribute__((unused)) = paddle::framework::OpRegistry::CreateOp(op_desc); } catch (paddle::framework::EnforceNotMet err) { caught = true; @@ -111,12 +163,11 @@ TEST(OpRegistry, CustomChecker) { attr->set_name("test_attr"); attr->set_type(paddle::framework::AttrType::INT); attr->set_i(4); - paddle::framework::OpBase* op = + paddle::framework::OperatorBase* op = paddle::framework::OpRegistry::CreateOp(op_desc); - std::string debug_str = op->Run(); - std::string str = "MyTestOp runs! test_attr = " + std::to_string(4); - ASSERT_EQ(str.size(), debug_str.size()); - for (size_t i = 0; i < debug_str.length(); ++i) { - ASSERT_EQ(debug_str[i], str[i]); - } + paddle::platform::CPUDeviceContext dev_ctx; + auto scope = std::make_shared(); + op->Run(scope, dev_ctx); + int test_attr = op->GetAttr("test_attr"); + ASSERT_EQ(test_attr, 4); } diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc new file mode 100644 index 0000000000000000000000000000000000000000..8f7adff8b3982e91a3d7f6d598cd62d5005d5f17 --- /dev/null +++ b/paddle/framework/operator.cc @@ -0,0 +1,43 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/operator.h" + +namespace paddle { +namespace framework { + +std::string OperatorBase::DebugString() const { + std::stringstream ss; + ss << "=================\n"; + ss << "type = " << desc_.type() << "\n"; + ss << "inputs = ["; + for (auto& ipt : inputs_) { + ss << ipt << ", "; + } + ss << "]\n"; + ss << "outputs = ["; + for (auto& opt : outputs_) { + ss << opt << ", "; + } + ss << "]\n"; + ss << "attr_keys = ["; + for (auto& attr : attrs_) { + ss << attr.first << ", "; + } + ss << "]\n"; + return ss.str(); +} + +} // namespace framework +} // namespace paddle \ No newline at end of file diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h new file mode 100644 index 0000000000000000000000000000000000000000..d3c55e0ceb6737772be38b536a107900ee895b12 --- /dev/null +++ b/paddle/framework/operator.h @@ -0,0 +1,185 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace paddle { +namespace framework { + +class OperatorBase; + +/** + * OperatorBase has the basic element that Net will call to do computation. + * Only CreateOperator from OpRegistry will new Operator directly. User + * should always construct a proto message OpDesc and call + * OpRegistry::CreateOp(op_desc) to get an Operator instance. + */ +class OperatorBase { + public: + virtual ~OperatorBase() {} + + template + inline const T& GetAttr(const std::string& name) const { + PADDLE_ENFORCE(attrs_.count(name) != 0, "%s should be in AttributeMap", + name); + return boost::get(attrs_.at(name)); + } + + std::string DebugString() const; + + /// Init will be called after CreateOperator, you can put some initialization + /// logic here. + virtual void Init() {} + + /// InferShape infer the size of Variables used by this Operator with + /// information inside scope + virtual void InferShape(const std::shared_ptr& scope) const = 0; + + /// Net will call this function to Run an op. + virtual void Run(const std::shared_ptr& scope, + const platform::DeviceContext& dev_ctx) const = 0; + + protected: + std::string Type() const { return desc_.type(); } + + public: + OpDesc desc_; + std::vector inputs_; + std::vector outputs_; + AttributeMap attrs_; +}; + +class OpKernel { + public: + /** + * KernelContext is the only parameter of Kernel Run function. + * Run will get input/output variables, state such as momentum and + * device resource such as CUDA stream, cublas handle, etc. from + * KernelContext. User should construct it before run the Operator. + */ + class KernelContext { + public: + KernelContext(const OperatorBase* op, const std::shared_ptr& scope, + const platform::DeviceContext& device_context) + : op_(*op), scope_(scope), device_context_(device_context) {} + + const Variable* Input(int index) const { + return scope_->GetVariable(op_.inputs_[index]); + } + + Variable* Output(int index) const { + return scope_->GetVariable(op_.outputs_[index]); + } + + const OperatorBase& op_; + const std::shared_ptr& scope_; + const platform::DeviceContext& device_context_; + }; + + virtual void Compute(const KernelContext& context) const = 0; + + virtual ~OpKernel() {} +}; + +template +struct VarToTensor {}; + +template <> +struct VarToTensor { + Tensor* operator()(Variable* var) { return var->GetMutable(); } +}; + +template <> +struct VarToTensor { + const Tensor* operator()(Variable* var) { return &var->Get(); } +}; + +class OperatorWithKernel : public OperatorBase { + public: + struct OpKernelKey { + platform::Place place_; + + OpKernelKey() = default; + OpKernelKey(const platform::DeviceContext& dev_ctx) { + place_ = dev_ctx.GetPlace(); + } + + bool operator==(const OpKernelKey& o) const { return place_ == o.place_; } + }; + + struct OpKernelHash { + std::hash hash_; + size_t operator()(const OpKernelKey& key) const { + return hash_(platform::is_gpu_place(key.place_)); + } + }; + + using OpKernelMap = + std::unordered_map, OpKernelHash>; + + void Run(const std::shared_ptr& scope, + const platform::DeviceContext& dev_ctx) const final { + auto& opKernel = AllOpKernels().at(Type()).at(OpKernelKey(dev_ctx)); + opKernel->Compute(OpKernel::KernelContext(this, scope, dev_ctx)); + } + + static std::unordered_map& + AllOpKernels() { + static std::unordered_map g_all_op_kernels; + return g_all_op_kernels; + } + void InferShape(const std::shared_ptr& scope) const final { + std::vector ins; + VarNamesToTensors(scope, inputs_, &ins); + std::vector outs; + VarNamesToTensors(scope, outputs_, &outs); + InferShape(ins, outs); + }; + + private: + template + void VarNamesToTensors(const std::shared_ptr& scope, + const std::vector& var_names, + std::vector* container) const { + container->reserve(var_names.size()); + VarToTensor convert; + for (auto& name : var_names) { + auto var = scope->GetVariable(name); + if (var != nullptr) { + container->push_back(convert(var)); + } else { + container->push_back(nullptr); + } + } + } + + protected: + virtual void InferShape(const std::vector& inputs, + const std::vector& outputs) const = 0; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..204b601a4aad52ee57b81235851c9806b607799f --- /dev/null +++ b/paddle/framework/operator_test.cc @@ -0,0 +1,94 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/operator.h" +#include "gtest/gtest.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace framework { + +class OperatorTest : public OperatorBase { + public: + void Init() override { x = 1; } + void InferShape(const std::shared_ptr& scope) const override {} + void Run(const std::shared_ptr& scope, + const platform::DeviceContext& dev_ctx) const override { + float scale = GetAttr("scale"); + ASSERT_NEAR(scale, 3.14, 1e-5); + ASSERT_EQ(scope->GetVariable(inputs_[0]), nullptr); + ASSERT_EQ(x, 1); + ASSERT_NE(scope->GetVariable(outputs_[0]), nullptr); + } + + public: + float x = 0; +}; + +class OpKernelTestProtoAndCheckerMaker : public OpProtoAndCheckerMaker { + public: + OpKernelTestProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of test op"); + AddOutput("output", "output of test op"); + AddAttr("scale", "scale of cosine op") + .SetDefault(1.0) + .LargerThan(0.0); + AddComment("This is test op"); + } +}; + +class OpWithKernelTest : public OperatorWithKernel { + protected: + void InferShape(const std::vector& inputs, + const std::vector& outputs) const override {} +}; + +class CPUKernelTest : public OpKernel { + public: + void Compute(const KernelContext& context) const { + float scale = context.op_.GetAttr("scale"); + ASSERT_NEAR(scale, 3.14, 1e-5); + std::cout << "this is cpu kernel" << std::endl; + std::cout << context.op_.DebugString() << std::endl; + } +}; + +} // namespace framework +} // namespace paddle + +REGISTER_OP(op_with_kernel, paddle::framework::OpWithKernelTest, + paddle::framework::OpKernelTestProtoAndCheckerMaker); +REGISTER_OP_CPU_KERNEL(op_with_kernel, paddle::framework::CPUKernelTest); + +TEST(OpKernel, all) { + using namespace paddle::framework; + + OpDesc op_desc; + op_desc.set_type("op_with_kernel"); + *op_desc.mutable_inputs()->Add() = "IN1"; + *op_desc.mutable_outputs()->Add() = "OUT1"; + auto attr = op_desc.mutable_attrs()->Add(); + attr->set_name("scale"); + attr->set_type(paddle::framework::AttrType::FLOAT); + attr->set_f(3.14); + + paddle::platform::CPUDeviceContext cpu_device_context; + auto scope = std::make_shared(); + + OperatorBase* op = paddle::framework::OpRegistry::CreateOp(op_desc); + op->Run(scope, cpu_device_context); + + delete op; +} diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h index ce5d98b04e6b53fcedc4fc4610d9390e64846b2a..a0945e8055625ca4c21ea1c3fa9f27321ca9ba3c 100644 --- a/paddle/framework/tensor.h +++ b/paddle/framework/tensor.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include #include #include "paddle/framework/ddim.h" @@ -26,31 +27,65 @@ namespace framework { class Tensor { public: + Tensor() : offset_(0) {} + + explicit Tensor(const DDim& dims) : dims_(dims), offset_(0) {} + template const T* data() const { - PADDLE_ENFORCE(holder_ != nullptr, - "Tensor::data must be called after Tensor::mutable_data."); - return static_cast(holder_->Ptr()); + PADDLE_ENFORCE( + holder_ != nullptr, + "Tenosr has not been initialized. Call Tensor::mutable_data first."); + return reinterpret_cast( + reinterpret_cast(holder_->Ptr()) + offset_); } template ::value>::type* = nullptr> T* mutable_data(DDim dims, paddle::platform::Place place) { + dims_ = dims; if (holder_ == nullptr || !(holder_->Place() == place) /* some versions of boost::variant don't have operator!= */ - || holder_->Size() < product(dims) * sizeof(T)) { + || holder_->Size() < product(dims) * sizeof(T) + offset_) { holder_.reset(new PlaceholderImpl(place, product(dims) * sizeof(T))); + offset_ = 0; } - return static_cast(holder_->Ptr()); + return reinterpret_cast(reinterpret_cast(holder_->Ptr()) + + offset_); } - template ::value>::type* = nullptr> - T* mutable_data(DDim dims) { - return mutable_data(dims, paddle::platform::get_place()); + void ShareDataFrom(const Tensor& src) { + PADDLE_ENFORCE(src.holder_ != nullptr, + "Can not share data from an uninitialized tensor."); + holder_ = src.holder_; + dims_ = src.dims_; + offset_ = src.offset_; } + Tensor Slice(const int& begin_idx, const int& end_idx) const { + PADDLE_ENFORCE(holder_ != nullptr, + "The sliced tenosr has not been initialized."); + PADDLE_ENFORCE(begin_idx >= 0 && end_idx <= dims_[0], + "Slice index is less than zero or out of bound."); + PADDLE_ENFORCE(begin_idx < end_idx, + "Begin index must be less than end index."); + PADDLE_ENFORCE(dims_[0] != 1, "Can not slice a tensor with dims_[0] = 1."); + std::vector d = vectorize(dims_); + int base = 1; + for (size_t i = 1; i < d.size(); ++i) { + base *= d[i]; + } + Tensor dst; + dst.holder_ = holder_; + dst.dims_ = dims_; + dst.dims_[0] = end_idx - begin_idx; + dst.offset_ = offset_ + begin_idx * base * holder_->TypeSize(); + return dst; + } + + DDim dims() const { return dims_; } + private: // Placeholder hides type T, so it doesn't appear as a template // parameter of Variable. @@ -59,6 +94,7 @@ class Tensor { virtual void* Ptr() const = 0; virtual paddle::platform::Place Place() const = 0; virtual size_t Size() const = 0; + virtual size_t TypeSize() const = 0; }; template @@ -85,6 +121,7 @@ class Tensor { virtual void* Ptr() const { return static_cast(ptr_.get()); } virtual size_t Size() const { return size_; } virtual paddle::platform::Place Place() const { return place_; } + virtual size_t TypeSize() const { return sizeof(T); } std::unique_ptr ptr_; paddle::platform::Place place_; // record the place of ptr_. @@ -92,6 +129,8 @@ class Tensor { }; std::shared_ptr holder_; // holds the memory block if allocated. + DDim dims_; + size_t offset_; // marks the begin of tensor data area. }; } // namespace framework diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc index 727d81f8d72e39ec564c42a48bf7ff64204adfff..f4822838cfbd27656232a23b14f716f2fbe510e0 100644 --- a/paddle/framework/tensor_test.cc +++ b/paddle/framework/tensor_test.cc @@ -15,15 +15,27 @@ #include #include -TEST(Tensor, ASSERT) { - paddle::framework::Tensor cpu_tensor; +TEST(Tensor, Dims) { + using namespace paddle::framework; + using namespace paddle::platform; + Tensor tt(make_ddim({2, 3, 4})); + DDim dims = tt.dims(); + ASSERT_EQ(arity(dims), 3); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(i + 2, dims[i]); + } +} + +TEST(Tensor, DataAssert) { + paddle::framework::Tensor src_tensor; bool caught = false; try { - const double* p __attribute__((unused)) = cpu_tensor.data(); + src_tensor.data(); } catch (paddle::framework::EnforceNotMet err) { caught = true; - std::string msg = "Tensor::data must be called after Tensor::mutable_data."; + std::string msg = + "Tenosr has not been initialized. Call Tensor::mutable_data first."; const char* what = err.what(); for (size_t i = 0; i < msg.length(); ++i) { ASSERT_EQ(what[i], msg[i]); @@ -32,54 +44,138 @@ TEST(Tensor, ASSERT) { ASSERT_TRUE(caught); } -/* mutable_data() is not tested at present +/* following tests are not available at present because Memory::Alloc() and Memory::Free() have not been ready. TEST(Tensor, MutableData) { using namespace paddle::framework; using namespace paddle::platform; { - Tensor cpu_tensor; + Tensor src_tensor; float* p1 = nullptr; float* p2 = nullptr; // initialization - p1 = cpu_tensor.mutable_data(make_ddim({1, 2, 3}), CPUPlace()); + p1 = src_tensor.mutable_data(make_ddim({1, 2, 3}), CPUPlace()); EXPECT_NE(p1, nullptr); - // set cpu_tensor a new dim with large size + // set src_tensor a new dim with large size // momery is supposed to be re-allocated - p2 = cpu_tensor.mutable_data(make_ddim({3, 4})); + p2 = src_tensor.mutable_data(make_ddim({3, 4}), CPUPlace()); EXPECT_NE(p2, nullptr); EXPECT_NE(p1, p2); - // set cpu_tensor a new dim with same size + // set src_tensor a new dim with same size // momery block is supposed to be unchanged - p1 = cpu_tensor.mutable_data(make_ddim({2, 2, 3})); + p1 = src_tensor.mutable_data(make_ddim({2, 2, 3}), CPUPlace()); EXPECT_EQ(p1, p2); - // set cpu_tensor a new dim with smaller size + // set src_tensor a new dim with smaller size // momery block is supposed to be unchanged - p2 = cpu_tensor.mutable_data(make_ddim({2, 2})); + p2 = src_tensor.mutable_data(make_ddim({2, 2}), CPUPlace()); EXPECT_EQ(p1, p2); } { - Tensor gpu_tensor; + Tensor src_tensor; float* p1 = nullptr; float* p2 = nullptr; // initialization - p1 = gpu_tensor.mutable_data(make_ddim({1, 2, 3}), GPUPlace()); + p1 = src_tensor.mutable_data(make_ddim({1, 2, 3}), GPUPlace()); EXPECT_NE(p1, nullptr); - // set gpu_tensor a new dim with large size + // set src_tensor a new dim with large size // momery is supposed to be re-allocated - p2 = gpu_tensor.mutable_data(make_ddim({3, 4})); + p2 = src_tensor.mutable_data(make_ddim({3, 4}), GPUPlace()); EXPECT_NE(p2, nullptr); EXPECT_NE(p1, p2); - // set gpu_tensor a new dim with same size + // set src_tensor a new dim with same size // momery block is supposed to be unchanged - p1 = gpu_tensor.mutable_data(make_ddim({2, 2, 3})); + p1 = src_tensor.mutable_data(make_ddim({2, 2, 3}), GPUPlace()); EXPECT_EQ(p1, p2); - // set gpu_tensor a new dim with smaller size + // set src_tensor a new dim with smaller size // momery block is supposed to be unchanged - p2 = gpu_tensor.mutable_data(make_ddim({2, 2})); + p2 = src_tensor.mutable_data(make_ddim({2, 2}), GPUPlace()); EXPECT_EQ(p1, p2); } } -*/ + +TEST(Tensor, ShareDataFrom) { + using namespace paddle::framework; + using namespace paddle::platform; + { + Tensor src_tensor; + Tensor dst_tensor; + // Try to share data form uninitialized tensor + bool caught = false; + try { + dst_tensor.ShareDataFrom(src_tensor); + } catch (EnforceNotMet err) { + caught = true; + std::string msg = "Can not share data from an uninitialized tensor."; + const char* what = err.what(); + for (size_t i = 0; i < msg.length(); ++i) { + ASSERT_EQ(what[i], msg[i]); + } + } + ASSERT_TRUE(caught); + + src_tensor.mutable_data(make_ddim({2, 3, 4}), CPUPlace()); + dst_tensor.ShareDataFrom(src_tensor); + ASSERT_EQ(src_tensor.data(), dst_tensor.data()); + } + + { + Tensor src_tensor; + Tensor dst_tensor; + src_tensor.mutable_data(make_ddim({2, 3, 4}), GPUPlace()); + dst_tensor.ShareDataFrom(src_tensor); + ASSERT_EQ(src_tensor.data(), dst_tensor.data()); + } +} + +TEST(Tensor, Slice) { + using namespace paddle::framework; + using namespace paddle::platform; + { + Tensor src_tensor; + src_tensor.mutable_data(make_ddim({5, 3, 4}), CPUPlace()); + Tensor slice_tensor = src_tensor.Slice(1, 3); + DDim slice_dims = slice_tensor.dims(); + ASSERT_EQ(arity(slice_dims), 3); + EXPECT_EQ(slice_dims[0], 2); + EXPECT_EQ(slice_dims[1], 3); + EXPECT_EQ(slice_dims[2], 4); + + uintptr_t src_data_address = + reinterpret_cast(src_tensor.data()); + uintptr_t src_mutable_data_address = reinterpret_cast( + src_tensor.mutable_data(src_tensor.dims(), CPUPlace())); + uintptr_t slice_data_address = + reinterpret_cast(slice_tensor.data()); + uintptr_t slice_mutable_data_address = reinterpret_cast( + slice_tensor.mutable_data(slice_tensor.dims(), CPUPlace())); + EXPECT_EQ(src_data_address, src_mutable_data_address); + EXPECT_EQ(slice_data_address, slice_mutable_data_address); + EXPECT_EQ(src_data_address + 3 * 4 * 1 * sizeof(int), slice_data_address); + } + + { + Tensor src_tensor; + src_tensor.mutable_data(make_ddim({6, 9}), GPUPlace()); + Tensor slice_tensor = src_tensor.Slice(2, 6); + DDim slice_dims = slice_tensor.dims(); + ASSERT_EQ(arity(slice_dims), 2); + EXPECT_EQ(slice_dims[0], 4); + EXPECT_EQ(slice_dims[1], 9); + + uintptr_t src_data_address = + reinterpret_cast(src_tensor.data()); + uintptr_t src_mutable_data_address = reinterpret_cast( + src_tensor.mutable_data(src_tensor.dims(), GPUPlace())); + uintptr_t slice_data_address = + reinterpret_cast(slice_tensor.data()); + uintptr_t slice_mutable_data_address = reinterpret_cast( + slice_tensor.mutable_data(slice_tensor.dims(), GPUPlace())); + EXPECT_EQ(src_data_address, src_mutable_data_address); + EXPECT_EQ(slice_data_address, slice_mutable_data_address); + EXPECT_EQ(src_data_address + 9 * 2 * sizeof(double), slice_data_address); + } +} + +*/ \ No newline at end of file diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..40bb326512c118178184120d4bc26dc127689ff3 --- /dev/null +++ b/paddle/operators/CMakeLists.txt @@ -0,0 +1,6 @@ +if(WITH_GPU) + nv_library(add_op SRCS add_op.cc add_op.cu DEPS operator op_registry glog ddim) +else() + cc_library(add_op SRCS add_op.cc DEPS operator op_registry glog ddim) +endif() +cc_test(add_op_test SRCS add_op_test.cc DEPS add_op) diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..2766f0bf258ed863a4297c1e4a2be4673cbf3044 --- /dev/null +++ b/paddle/operators/add_op.cc @@ -0,0 +1,44 @@ +#include +#include +#include + +namespace paddle { +namespace operators { + +class AddOp : public framework::OperatorWithKernel { +protected: + void InferShape( + const std::vector &inputs, + const std::vector &outputs) const override { + PADDLE_ENFORCE(inputs.size() == 2, "Input size of AddOp must be two"); + PADDLE_ENFORCE(outputs.size() == 1, "Output size of AddOp must be one"); + PADDLE_ENFORCE( + inputs[0] != nullptr && inputs[1] != nullptr && outputs[0] != nullptr, + "Inputs/Outputs of AddOp must all be set"); + PADDLE_ENFORCE(inputs[0]->dims() == inputs[1]->dims(), + "Two input of Add Op's dimension must be same."); + // Need set dims in Tensor + // outputs[0]->set_dims(inputs[0]->dims()) + } +}; + +class AddOpMaker : public framework::OpProtoAndCheckerMaker { +public: + AddOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : framework::OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The first input of add op"); + AddInput("Y", "The second input of add op"); + AddOutput("Out", "The output of add op"); + AddComment(R"DOC( +Two Element Add Operator. + +The equation is: Out = X + Y +)DOC"); + } +}; +} // namespace op +} // namespace paddle + +REGISTER_OP(add_two, paddle::operators::AddOp, paddle::operators::AddOpMaker); +REGISTER_OP_CPU_KERNEL( + add_two, ::paddle::operators::AddKernel<::paddle::platform::CPUPlace>); \ No newline at end of file diff --git a/paddle/operators/add_op.cu b/paddle/operators/add_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..5979345fffd68d71ba09dc96874d8ff9471bdbcc --- /dev/null +++ b/paddle/operators/add_op.cu @@ -0,0 +1,5 @@ +#include +#include + +REGISTER_OP_GPU_KERNEL(add_two, + paddle::operators::AddKernel); \ No newline at end of file diff --git a/paddle/operators/add_op.h b/paddle/operators/add_op.h new file mode 100644 index 0000000000000000000000000000000000000000..17d459dbc86af73fa86934a5ccce9da509c59c6b --- /dev/null +++ b/paddle/operators/add_op.h @@ -0,0 +1,17 @@ +#pragma once +#include +#include + +namespace paddle { +namespace operators { + +template +class AddKernel : public framework::OpKernel { +public: + void Compute(const KernelContext &context) const override { + LOG(INFO) << "Add kernel in " << typeid(Place).name(); + } +}; + +} // namespace op +} // namespace paddle diff --git a/paddle/operators/add_op_test.cc b/paddle/operators/add_op_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..f554ac1bef3255f136ad4407a7a1096bdc2b1db5 --- /dev/null +++ b/paddle/operators/add_op_test.cc @@ -0,0 +1,9 @@ +#include +#define private public +#include +USE_OP(add_two); +TEST(AddOp, GetOpProto) { + auto& protos = paddle::framework::OpRegistry::protos(); + auto it = protos.find("add_two"); + ASSERT_NE(it, protos.end()); +} \ No newline at end of file diff --git a/paddle/optimizer/optimizer.cc b/paddle/optimizer/optimizer.cc index 54662dc37891d3211950453b210db4b475837df4..eb7125adee769c97e16986cabf06ea389bf4c143 100644 --- a/paddle/optimizer/optimizer.cc +++ b/paddle/optimizer/optimizer.cc @@ -44,8 +44,8 @@ paddle_optimizer* paddle_create_optimizer(const unsigned char* config_proto, const int state_len) { paddle_optimizer* optimizer = new paddle_optimizer; std::string config(config_proto, config_proto + config_proto_len); - Tensor* parameter = - new Tensor(reinterpret_cast(param_buffer), num_bytes); + Tensor* parameter = new Tensor(reinterpret_cast(param_buffer), + num_bytes / sizeof(float)); optimizer->impl = ParameterOptimizer::Create(config, parameter); if (state != nullptr) { std::string s(state, state + state_len); @@ -65,7 +65,8 @@ int paddle_update_parameter(paddle_optimizer* o, int num_bytes) { // TOOD(zhihong): datatype not work. need to add the runtime datatype auto grad_type = reinterpret_cast(grad_buffer); - Tensor* gradient = new Tensor(const_cast(grad_type), num_bytes); + Tensor* gradient = + new Tensor(const_cast(grad_type), num_bytes / sizeof(float)); o->impl->Update(gradient); return PADDLE_SUCCESS; } diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt index d16c747aee2f9dede4256fe9a99aec881fe648f8..6ac4035c0f863c5f63d17b523a7a8be668ff3da0 100644 --- a/paddle/platform/CMakeLists.txt +++ b/paddle/platform/CMakeLists.txt @@ -6,6 +6,13 @@ nv_library(gpu_info SRCS gpu_info.cc DEPS gflags) cc_library(place SRCS place.cc) cc_test(place_test SRCS place_test.cc DEPS place glog gflags) -cc_library(dynamic_loader SRCS dynload/dynamic_loader.cc DEPS gflags glog) +add_subdirectory(dynload) -nv_test(device_context_test SRCS device_context_test.cc DEPS dynamic_loader place eigen3 gpu_info) +IF(WITH_GPU) + set(GPU_CTX_DEPS dynload_cuda dynamic_loader) +ELSE() + set(GPU_CTX_DEPS) +ENDIF() + +cc_library(device_context SRCS device_context.cc DEPS place eigen3 ${GPU_CTX_DEPS}) +nv_test(device_context_test SRCS device_context_test.cc DEPS device_context gpu_info) diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc new file mode 100644 index 0000000000000000000000000000000000000000..960ef0a5955bfe5f7d33b7c8e4524176b0dbfda6 --- /dev/null +++ b/paddle/platform/device_context.cc @@ -0,0 +1,30 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/platform/device_context.h" + +namespace paddle { +namespace platform { + +template <> +Eigen::DefaultDevice* DeviceContext::get_eigen_device() { + return reinterpret_cast(this)->eigen_device(); +} + +#ifndef PADDLE_ONLY_CPU +template <> +Eigen::GpuDevice* DeviceContext::get_eigen_device() { + return reinterpret_cast(this)->eigen_device(); +} +#endif + +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index 02194581d1dcb1c9d117dde44a45de94e65ae475..51c8e1391324649d8e845e902a5632f6bca1fa58 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -1,11 +1,8 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,8 +20,9 @@ limitations under the License. */ #include "paddle/platform/gpu_info.h" #define EIGEN_USE_GPU #endif -#include "paddle/platform/place.h" -#include "unsupported/Eigen/CXX11/Tensor" +#include +#include +#include namespace paddle { namespace platform { @@ -32,9 +30,29 @@ namespace platform { class DeviceContext { public: virtual ~DeviceContext() {} + virtual Place GetPlace() const = 0; + + template + DeviceType* get_eigen_device(); }; -class CPUDeviceContext : public DeviceContext {}; +class CPUDeviceContext : public DeviceContext { + public: + Eigen::DefaultDevice* eigen_device() { + if (!eigen_device_) { + eigen_device_.reset(new Eigen::DefaultDevice()); + } + return eigen_device_.get(); + } + + Place GetPlace() const override { + Place retv = CPUPlace(); + return retv; + } + + private: + std::unique_ptr eigen_device_; +}; #ifndef PADDLE_ONLY_CPU @@ -58,8 +76,13 @@ class CUDADeviceContext : public DeviceContext { GPUPlaceGuard guard(gpu_place_); paddle::platform::throw_on_error(cudaStreamCreate(&stream_), "cudaStreamCreate failed"); - eigen_stream_ = new Eigen::CudaStreamDevice(&stream_); - eigen_device_ = new Eigen::GpuDevice(eigen_stream_); + eigen_stream_.reset(new Eigen::CudaStreamDevice(&stream_)); + eigen_device_.reset(new Eigen::GpuDevice(eigen_stream_.get())); + } + + Place GetPlace() const override { + Place retv = GPUPlace(); + return retv; } void Wait() { @@ -69,7 +92,7 @@ class CUDADeviceContext : public DeviceContext { cudaStream_t stream() { return stream_; } - Eigen::GpuDevice eigen_device() { return *eigen_device_; } + Eigen::GpuDevice* eigen_device() { return eigen_device_.get(); } cublasHandle_t cublas_handle() { if (!blas_handle_) { @@ -134,10 +157,8 @@ class CUDADeviceContext : public DeviceContext { rand_generator_) == CURAND_STATUS_SUCCESS, "curandDestroyGenerator failed"); } - - delete eigen_stream_; - delete eigen_device_; - + eigen_stream_.reset(); + eigen_device_.reset(); paddle::platform::throw_on_error(cudaStreamDestroy(stream_), "cudaStreamDestroy failed"); } @@ -146,8 +167,8 @@ class CUDADeviceContext : public DeviceContext { GPUPlace gpu_place_; cudaStream_t stream_; - Eigen::CudaStreamDevice* eigen_stream_; - Eigen::GpuDevice* eigen_device_; + std::unique_ptr eigen_stream_; + std::unique_ptr eigen_device_; cublasHandle_t blas_handle_{nullptr}; @@ -156,6 +177,8 @@ class CUDADeviceContext : public DeviceContext { int random_seed_; curandGenerator_t rand_generator_{nullptr}; }; + #endif + } // namespace platform } // namespace paddle diff --git a/paddle/platform/device_context_test.cc b/paddle/platform/device_context_test.cc index 61be4a307dbf073be7dff4564183240834cc7df6..af2ce17fc2238dda62e9888ebe9426edcd55d2bc 100644 --- a/paddle/platform/device_context_test.cc +++ b/paddle/platform/device_context_test.cc @@ -15,13 +15,26 @@ limitations under the License. */ #include "paddle/platform/device_context.h" #include "gtest/gtest.h" -TEST(CUDADeviceContext, Init) { +using DEVICE_GPU = Eigen::GpuDevice; +TEST(Device, Init) { + int count = paddle::platform::GetDeviceCount(); + for (int i = 0; i < count; i++) { + paddle::platform::DeviceContext* device_context = + new paddle::platform::CUDADeviceContext(i); + Eigen::GpuDevice* gpu_device = + device_context->template get_eigen_device(); + ASSERT_NE(nullptr, gpu_device); + delete device_context; + } +} + +TEST(Device, CUDADeviceContext) { int count = paddle::platform::GetDeviceCount(); for (int i = 0; i < count; i++) { paddle::platform::CUDADeviceContext* device_context = new paddle::platform::CUDADeviceContext(i); - Eigen::GpuDevice gpu_device = device_context->eigen_device(); - ASSERT_NE(nullptr, gpu_device.stream()); + Eigen::GpuDevice* gpu_device = device_context->eigen_device(); + ASSERT_NE(nullptr, gpu_device); cudnnHandle_t cudnn_handle = device_context->cudnn_handle(); ASSERT_NE(nullptr, cudnn_handle); cublasHandle_t cublas_handle = device_context->cublas_handle(); diff --git a/paddle/platform/dynload/CMakeLists.txt b/paddle/platform/dynload/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d205ead84598e04eea523be32139959a02e0dd83 --- /dev/null +++ b/paddle/platform/dynload/CMakeLists.txt @@ -0,0 +1,2 @@ +cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags) +nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc) diff --git a/paddle/platform/dynload/cublas.cc b/paddle/platform/dynload/cublas.cc new file mode 100644 index 0000000000000000000000000000000000000000..4e3dfdaefb2348346e8f917b1f6c758bf6d91a1a --- /dev/null +++ b/paddle/platform/dynload/cublas.cc @@ -0,0 +1,15 @@ +#include + +namespace paddle { +namespace platform { +namespace dynload { +std::once_flag cublas_dso_flag; +void *cublas_dso_handle = nullptr; + +#define DEFINE_WRAP(__name) DynLoad__##__name __name + +CUBLAS_BLAS_ROUTINE_EACH(DEFINE_WRAP); + +} // namespace dynload +} // namespace platform +} // namespace paddle diff --git a/paddle/platform/dynload/cublas.h b/paddle/platform/dynload/cublas.h index 258cc88031a71e9fee65b5445bd5537d6782e226..c44b7240a885c2ef71e550df645dbaded69f9944 100644 --- a/paddle/platform/dynload/cublas.h +++ b/paddle/platform/dynload/cublas.h @@ -23,8 +23,8 @@ namespace paddle { namespace platform { namespace dynload { -std::once_flag cublas_dso_flag; -void *cublas_dso_handle = nullptr; +extern std::once_flag cublas_dso_flag; +extern void *cublas_dso_handle; /** * The following macro definition can generate structs @@ -34,10 +34,10 @@ void *cublas_dso_handle = nullptr; * note: default dynamic linked libs */ #ifdef PADDLE_USE_DSO -#define DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ +#define DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ struct DynLoad__##__name { \ template \ - cublasStatus_t operator()(Args... args) { \ + inline cublasStatus_t operator()(Args... args) { \ typedef cublasStatus_t (*cublasFunc)(Args...); \ std::call_once(cublas_dso_flag, \ paddle::platform::dynload::GetCublasDsoHandle, \ @@ -45,62 +45,46 @@ void *cublas_dso_handle = nullptr; void *p_##__name = dlsym(cublas_dso_handle, #__name); \ return reinterpret_cast(p_##__name)(args...); \ } \ - } __name; // struct DynLoad__##__name + }; \ + extern DynLoad__##__name __name #else -#define DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - cublasStatus_t operator()(Args... args) { \ - return __name(args...); \ - } \ - } __name; // struct DynLoad__##__name +#define DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) \ + struct DynLoad__##__name { \ + inline template \ + cublasStatus_t operator()(Args... args) { \ + return __name(args...); \ + } \ + }; \ + extern DynLoad__##__name __name #endif -#define DYNAMIC_LOAD_CUBLAS_V2_WRAP(__name) DYNAMIC_LOAD_CUBLAS_WRAP(__name) +#define DECLARE_DYNAMIC_LOAD_CUBLAS_V2_WRAP(__name) \ + DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP(__name) -// include all needed cublas functions in HPPL -// clang-format off #define CUBLAS_BLAS_ROUTINE_EACH(__macro) \ - __macro(cublasSgemv) \ - __macro(cublasDgemv) \ - __macro(cublasSgemm) \ - __macro(cublasDgemm) \ - __macro(cublasSgeam) \ - __macro(cublasDgeam) \ + __macro(cublasSgemv); \ + __macro(cublasDgemv); \ + __macro(cublasSgemm); \ + __macro(cublasDgemm); \ + __macro(cublasSgeam); \ + __macro(cublasDgeam); \ + __macro(cublasCreate_v2); \ + __macro(cublasDestroy_v2); \ + __macro(cublasSetStream_v2); \ + __macro(cublasSetPointerMode_v2); \ + __macro(cublasGetPointerMode_v2); \ + __macro(cublasSgemmBatched); \ + __macro(cublasDgemmBatched); \ + __macro(cublasCgemmBatched); \ + __macro(cublasZgemmBatched); \ + __macro(cublasSgetrfBatched); \ + __macro(cublasSgetriBatched); \ + __macro(cublasDgetrfBatched); \ + __macro(cublasDgetriBatched) -DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasCreate) -DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasDestroy) -DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasSetStream) -DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasSetPointerMode) -DYNAMIC_LOAD_CUBLAS_V2_WRAP(cublasGetPointerMode) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgemmBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgemmBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasCgemmBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasZgemmBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetrfBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetriBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetrfBatched) -DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetriBatched) -CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) +CUBLAS_BLAS_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP); -#undef DYNAMIC_LOAD_CUBLAS_WRAP -#undef DYNAMIC_LOAD_CUBLAS_V2_WRAP -#undef CUBLAS_BLAS_ROUTINE_EACH - -// clang-format on -#ifndef PADDLE_TYPE_DOUBLE -#define CUBLAS_GEAM paddle::platform::dynload::cublasSgeam -#define CUBLAS_GEMV paddle::platform::dynload::cublasSgemv -#define CUBLAS_GEMM paddle::platform::dynload::cublasSgemm -#define CUBLAS_GETRF paddle::platform::dynload::cublasSgetrfBatched -#define CUBLAS_GETRI paddle::platform::dynload::cublasSgetriBatched -#else -#define CUBLAS_GEAM paddle::platform::dynload::cublasDgeam -#define CUBLAS_GEMV paddle::platform::dynload::cublasDgemv -#define CUBLAS_GEMM paddle::platform::dynload::cublasDgemm -#define CUBLAS_GETRF paddle::platform::dynload::cublasDgetrfBatched -#define CUBLAS_GETRI paddle::platform::dynload::cublasDgetriBatched -#endif +#undef DECLARE_DYNAMIC_LOAD_CUBLAS_WRAP } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/platform/dynload/cudnn.cc b/paddle/platform/dynload/cudnn.cc new file mode 100644 index 0000000000000000000000000000000000000000..8b5e15b5efcdae6a1eed09f002eb2f4f2163035f --- /dev/null +++ b/paddle/platform/dynload/cudnn.cc @@ -0,0 +1,28 @@ +#include + +namespace paddle { +namespace platform { +namespace dynload { +std::once_flag cudnn_dso_flag; +void* cudnn_dso_handle = nullptr; + +#define DEFINE_WRAP(__name) DynLoad__##__name __name + +CUDNN_DNN_ROUTINE_EACH(DEFINE_WRAP); +CUDNN_DNN_ROUTINE_EACH_R2(DEFINE_WRAP); + +#ifdef CUDNN_DNN_ROUTINE_EACH_AFTER_R3 +CUDNN_DNN_ROUTINE_EACH_AFTER_R3(DEFINE_WRAP); +#endif + +#ifdef CUDNN_DNN_ROUTINE_EACH_AFTER_R4 +CUDNN_DNN_ROUTINE_EACH_AFTER_R4(DEFINE_WRAP); +#endif + +#ifdef CUDNN_DNN_ROUTINE_EACH_R5 +CUDNN_DNN_ROUTINE_EACH_R5(DEFINE_WRAP); +#endif + +} // namespace dynload +} // namespace platform +} // namespace paddle \ No newline at end of file diff --git a/paddle/platform/dynload/cudnn.h b/paddle/platform/dynload/cudnn.h index 0a9562c573cdfe059ef7caa39ba62efa87225e41..ef0dd85b083dc2335dd5c70d3dc5f59eda25daeb 100644 --- a/paddle/platform/dynload/cudnn.h +++ b/paddle/platform/dynload/cudnn.h @@ -23,12 +23,12 @@ namespace paddle { namespace platform { namespace dynload { -std::once_flag cudnn_dso_flag; -void* cudnn_dso_handle = nullptr; +extern std::once_flag cudnn_dso_flag; +extern void* cudnn_dso_handle; #ifdef PADDLE_USE_DSO -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ +#define DECLARE_DYNAMIC_LOAD_CUDNN_WRAP(__name) \ struct DynLoad__##__name { \ template \ auto operator()(Args... args) -> decltype(__name(args...)) { \ @@ -39,17 +39,19 @@ void* cudnn_dso_handle = nullptr; void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ return reinterpret_cast(p_##__name)(args...); \ } \ - } __name; /* struct DynLoad__##__name */ + }; \ + extern struct DynLoad__##__name __name #else -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ +#define DECLARE_DYNAMIC_LOAD_CUDNN_WRAP(__name) \ struct DynLoad__##__name { \ template \ auto operator()(Args... args) -> decltype(__name(args...)) { \ return __name(args...); \ } \ - } __name; /* struct DynLoad__##__name */ + }; \ + extern DynLoad__##__name __name #endif @@ -57,80 +59,73 @@ void* cudnn_dso_handle = nullptr; * include all needed cudnn functions in HPPL * different cudnn version has different interfaces **/ -// clang-format off -#define CUDNN_DNN_ROUTINE_EACH(__macro) \ - __macro(cudnnSetTensor4dDescriptor) \ - __macro(cudnnSetTensor4dDescriptorEx) \ - __macro(cudnnGetConvolutionNdForwardOutputDim) \ - __macro(cudnnGetConvolutionForwardAlgorithm) \ - __macro(cudnnCreateTensorDescriptor) \ - __macro(cudnnDestroyTensorDescriptor) \ - __macro(cudnnCreateFilterDescriptor) \ - __macro(cudnnSetFilter4dDescriptor) \ - __macro(cudnnSetPooling2dDescriptor) \ - __macro(cudnnDestroyFilterDescriptor) \ - __macro(cudnnCreateConvolutionDescriptor) \ - __macro(cudnnCreatePoolingDescriptor) \ - __macro(cudnnDestroyPoolingDescriptor) \ - __macro(cudnnSetConvolution2dDescriptor) \ - __macro(cudnnDestroyConvolutionDescriptor) \ - __macro(cudnnCreate) \ - __macro(cudnnDestroy) \ - __macro(cudnnSetStream) \ - __macro(cudnnActivationForward) \ - __macro(cudnnConvolutionForward) \ - __macro(cudnnConvolutionBackwardBias) \ - __macro(cudnnGetConvolutionForwardWorkspaceSize) \ - __macro(cudnnTransformTensor) \ - __macro(cudnnPoolingForward) \ - __macro(cudnnPoolingBackward) \ - __macro(cudnnSoftmaxBackward) \ - __macro(cudnnSoftmaxForward) \ - __macro(cudnnGetVersion) \ - __macro(cudnnGetErrorString) -CUDNN_DNN_ROUTINE_EACH(DYNAMIC_LOAD_CUDNN_WRAP) - -#define CUDNN_DNN_ROUTINE_EACH_R2(__macro) \ - __macro(cudnnAddTensor) \ - __macro(cudnnConvolutionBackwardData) \ - __macro(cudnnConvolutionBackwardFilter) -CUDNN_DNN_ROUTINE_EACH_R2(DYNAMIC_LOAD_CUDNN_WRAP) +#define CUDNN_DNN_ROUTINE_EACH(__macro) \ + __macro(cudnnSetTensor4dDescriptor); \ + __macro(cudnnSetTensor4dDescriptorEx); \ + __macro(cudnnGetConvolutionNdForwardOutputDim); \ + __macro(cudnnGetConvolutionForwardAlgorithm); \ + __macro(cudnnCreateTensorDescriptor); \ + __macro(cudnnDestroyTensorDescriptor); \ + __macro(cudnnCreateFilterDescriptor); \ + __macro(cudnnSetFilter4dDescriptor); \ + __macro(cudnnSetPooling2dDescriptor); \ + __macro(cudnnDestroyFilterDescriptor); \ + __macro(cudnnCreateConvolutionDescriptor); \ + __macro(cudnnCreatePoolingDescriptor); \ + __macro(cudnnDestroyPoolingDescriptor); \ + __macro(cudnnSetConvolution2dDescriptor); \ + __macro(cudnnDestroyConvolutionDescriptor); \ + __macro(cudnnCreate); \ + __macro(cudnnDestroy); \ + __macro(cudnnSetStream); \ + __macro(cudnnActivationForward); \ + __macro(cudnnConvolutionForward); \ + __macro(cudnnConvolutionBackwardBias); \ + __macro(cudnnGetConvolutionForwardWorkspaceSize); \ + __macro(cudnnTransformTensor); \ + __macro(cudnnPoolingForward); \ + __macro(cudnnPoolingBackward); \ + __macro(cudnnSoftmaxBackward); \ + __macro(cudnnSoftmaxForward); \ + __macro(cudnnGetVersion); \ + __macro(cudnnGetErrorString); +CUDNN_DNN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) + +#define CUDNN_DNN_ROUTINE_EACH_R2(__macro) \ + __macro(cudnnAddTensor); \ + __macro(cudnnConvolutionBackwardData); \ + __macro(cudnnConvolutionBackwardFilter); +CUDNN_DNN_ROUTINE_EACH_R2(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) // APIs available after R3: #if CUDNN_VERSION >= 3000 -#define CUDNN_DNN_ROUTINE_EACH_AFTER_R3(__macro) \ - __macro(cudnnGetConvolutionBackwardFilterWorkspaceSize) \ - __macro(cudnnGetConvolutionBackwardDataAlgorithm) \ - __macro(cudnnGetConvolutionBackwardFilterAlgorithm) \ - __macro(cudnnGetConvolutionBackwardDataWorkspaceSize) -CUDNN_DNN_ROUTINE_EACH_AFTER_R3(DYNAMIC_LOAD_CUDNN_WRAP) -#undef CUDNN_DNN_ROUTINE_EACH_AFTER_R3 +#define CUDNN_DNN_ROUTINE_EACH_AFTER_R3(__macro) \ + __macro(cudnnGetConvolutionBackwardFilterWorkspaceSize); \ + __macro(cudnnGetConvolutionBackwardDataAlgorithm); \ + __macro(cudnnGetConvolutionBackwardFilterAlgorithm); \ + __macro(cudnnGetConvolutionBackwardDataWorkspaceSize); +CUDNN_DNN_ROUTINE_EACH_AFTER_R3(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #endif - // APIs available after R4: #if CUDNN_VERSION >= 4007 -#define CUDNN_DNN_ROUTINE_EACH_AFTER_R4(__macro) \ - __macro(cudnnBatchNormalizationForwardTraining) \ - __macro(cudnnBatchNormalizationForwardInference) \ - __macro(cudnnBatchNormalizationBackward) -CUDNN_DNN_ROUTINE_EACH_AFTER_R4(DYNAMIC_LOAD_CUDNN_WRAP) -#undef CUDNN_DNN_ROUTINE_EACH_AFTER_R4 +#define CUDNN_DNN_ROUTINE_EACH_AFTER_R4(__macro) \ + __macro(cudnnBatchNormalizationForwardTraining); \ + __macro(cudnnBatchNormalizationForwardInference); \ + __macro(cudnnBatchNormalizationBackward); +CUDNN_DNN_ROUTINE_EACH_AFTER_R4(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #endif // APIs in R5 #if CUDNN_VERSION >= 5000 -#define CUDNN_DNN_ROUTINE_EACH_R5(__macro) \ - __macro(cudnnCreateActivationDescriptor) \ - __macro(cudnnSetActivationDescriptor) \ - __macro(cudnnGetActivationDescriptor) \ - __macro(cudnnDestroyActivationDescriptor) -CUDNN_DNN_ROUTINE_EACH_R5(DYNAMIC_LOAD_CUDNN_WRAP) -#undef CUDNN_DNN_ROUTINE_EACH_R5 +#define CUDNN_DNN_ROUTINE_EACH_R5(__macro) \ + __macro(cudnnCreateActivationDescriptor); \ + __macro(cudnnSetActivationDescriptor); \ + __macro(cudnnGetActivationDescriptor); \ + __macro(cudnnDestroyActivationDescriptor); +CUDNN_DNN_ROUTINE_EACH_R5(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP) #endif -#undef CUDNN_DNN_ROUTINE_EACH -// clang-format on } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/platform/dynload/curand.cc b/paddle/platform/dynload/curand.cc new file mode 100644 index 0000000000000000000000000000000000000000..5c1fab992c98569d4a95b6e699d97d428511e48e --- /dev/null +++ b/paddle/platform/dynload/curand.cc @@ -0,0 +1,15 @@ +#include + +namespace paddle { +namespace platform { +namespace dynload { + +std::once_flag curand_dso_flag; +void *curand_dso_handle; + +#define DEFINE_WRAP(__name) DynLoad__##__name __name + +CURAND_RAND_ROUTINE_EACH(DEFINE_WRAP); +} +} +} \ No newline at end of file diff --git a/paddle/platform/dynload/curand.h b/paddle/platform/dynload/curand.h index 9dc0a25c0fbdc3f73f1dd82206e8940972b0b7f5..d8c46bc41e18d013a80cd0a9116a4b1a52bf5854 100644 --- a/paddle/platform/dynload/curand.h +++ b/paddle/platform/dynload/curand.h @@ -22,10 +22,10 @@ limitations under the License. */ namespace paddle { namespace platform { namespace dynload { -std::once_flag curand_dso_flag; -void *curand_dso_handle = nullptr; +extern std::once_flag curand_dso_flag; +extern void *curand_dso_handle; #ifdef PADDLE_USE_DSO -#define DYNAMIC_LOAD_CURAND_WRAP(__name) \ +#define DECLARE_DYNAMIC_LOAD_CURAND_WRAP(__name) \ struct DynLoad__##__name { \ template \ curandStatus_t operator()(Args... args) { \ @@ -36,32 +36,29 @@ void *curand_dso_handle = nullptr; void *p_##__name = dlsym(curand_dso_handle, #__name); \ return reinterpret_cast(p_##__name)(args...); \ } \ - } __name; /* struct DynLoad__##__name */ + }; \ + extern DynLoad__##__name __name #else -#define DYNAMIC_LOAD_CURAND_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - curandStatus_t operator()(Args... args) { \ - return __name(args...); \ - } \ - } __name; /* struct DynLoad__##__name */ +#define DECLARE_DYNAMIC_LOAD_CURAND_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + curandStatus_t operator()(Args... args) { \ + return __name(args...); \ + } \ + }; \ + extern DynLoad__##__name __name #endif -/* include all needed curand functions in HPPL */ -// clang-format off -#define CURAND_RAND_ROUTINE_EACH(__macro) \ - __macro(curandCreateGenerator) \ - __macro(curandSetStream) \ - __macro(curandSetPseudoRandomGeneratorSeed)\ - __macro(curandGenerateUniform) \ - __macro(curandGenerateUniformDouble) \ - __macro(curandDestroyGenerator) -// clang-format on +#define CURAND_RAND_ROUTINE_EACH(__macro) \ + __macro(curandCreateGenerator); \ + __macro(curandSetStream); \ + __macro(curandSetPseudoRandomGeneratorSeed); \ + __macro(curandGenerateUniform); \ + __macro(curandGenerateUniformDouble); \ + __macro(curandDestroyGenerator); -CURAND_RAND_ROUTINE_EACH(DYNAMIC_LOAD_CURAND_WRAP) +CURAND_RAND_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CURAND_WRAP); -#undef CURAND_RAND_ROUTINE_EACH -#undef DYNAMIC_LOAD_CURAND_WRAP } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..af85fdeecb57729d7fb580ebd4c59c1afc61d61a --- /dev/null +++ b/paddle/pybind/CMakeLists.txt @@ -0,0 +1 @@ +cc_library(paddle_pybind SHARED SRCS pybind.cc DEPS pybind python) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc new file mode 100644 index 0000000000000000000000000000000000000000..f9f87acf15a6b62c343cc0e3db9ebc7e0aabb786 --- /dev/null +++ b/paddle/pybind/pybind.cc @@ -0,0 +1,47 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include + +namespace py = pybind11; +namespace pd = paddle::framework; + +PYBIND11_PLUGIN(core) { + py::module m("core", "C++ core of Paddle Paddle"); + + py::class_(m, "Variable", R"DOC(Variable Class. + +All parameter, weight, gradient are variables in Paddle. +)DOC") + .def("is_int", [](const pd::Variable& var) { return var.IsType(); }) + .def("set_int", + [](pd::Variable& var, int val) -> void { + *var.GetMutable() = val; + }) + .def("get_int", + [](const pd::Variable& var) -> int { return var.Get(); }); + + py::class_>(m, "Scope") + .def(py::init&>()) + .def("get_var", + &pd::Scope::GetVariable, + py::return_value_policy::reference) + .def("create_var", + &pd::Scope::CreateVariable, + py::return_value_policy::reference); + + return m.ptr(); +} diff --git a/paddle/scripts/travis/build_android.sh b/paddle/scripts/travis/build_android.sh new file mode 100755 index 0000000000000000000000000000000000000000..004067a8f55351509caaf2bbf6d5c349a4698a79 --- /dev/null +++ b/paddle/scripts/travis/build_android.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +ANDROID_STANDALONE_TOOLCHAIN=$HOME/android-toolchain-gcc +TMP_DIR=$HOME/$JOB/tmp +mkdir -p $TMP_DIR +cd $TMP_DIR +wget -q https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip +unzip -q android-ndk-r14b-linux-x86_64.zip +chmod +x $TMP_DIR/android-ndk-r14b/build/tools/make-standalone-toolchain.sh +$TMP_DIR/android-ndk-r14b/build/tools/make-standalone-toolchain.sh --force --arch=arm --platform=android-21 --install-dir=$ANDROID_STANDALONE_TOOLCHAIN +cd $HOME +rm -rf $TMP_DIR + +# Create the build directory for CMake. +mkdir -p $TRAVIS_BUILD_DIR/build_android +cd $TRAVIS_BUILD_DIR/build_android + +# Compile paddle binaries +cmake -DCMAKE_SYSTEM_NAME=Android \ + -DANDROID_STANDALONE_TOOLCHAIN=$ANDROID_STANDALONE_TOOLCHAIN \ + -DANDROID_ABI=armeabi-v7a \ + -DANDROID_ARM_NEON=ON \ + -DANDROID_ARM_MODE=ON \ + -DWITH_C_API=ON \ + -DWITH_SWIG_PY=OFF \ + -DWITH_STYLE_CHECK=OFF \ + .. + +make -j `nproc` diff --git a/paddle/trainer/NewRemoteParameterUpdater.cpp b/paddle/trainer/NewRemoteParameterUpdater.cpp index f25ce2f7f06f6da0feab27da61b8e49689cbe213..b359d9da2167bf459504e15c3140b3d956f417f3 100644 --- a/paddle/trainer/NewRemoteParameterUpdater.cpp +++ b/paddle/trainer/NewRemoteParameterUpdater.cpp @@ -22,7 +22,8 @@ DECLARE_string(save_dir); namespace paddle { NewRemoteParameterUpdater::NewRemoteParameterUpdater( const OptimizationConfig &config, const std::string pserverSpec) - : parameterClient_(-1), + : trainerConfig_(config), + parameterClient_(-1), newParameters_(nullptr), newGradients_(nullptr), pserverSpec_(pserverSpec) {} @@ -51,7 +52,22 @@ void NewRemoteParameterUpdater::init( LOG(INFO) << "paddle_begin_init_params start"; for (int i = 0; i < parameterSize(); ++i) { auto paramConfig = parameters_[i]->getConfig(); - std::string bytes = paramConfig.SerializeAsString(); + LOG(INFO) << "old param config: " << paramConfig.DebugString(); + // FIXME(typhoonzero): convert old paramConfig to optimizerConfig + OptimizerConfig optimizeConfigV2; + auto sgdConfigV2 = optimizeConfigV2.mutable_sgd(); + sgdConfigV2->set_momentum(paramConfig.momentum()); + sgdConfigV2->set_decay(paramConfig.decay_rate()); + optimizeConfigV2.set_lr_policy(paddle::OptimizerConfig::Const); + auto constlr = optimizeConfigV2.mutable_const_lr(); + constlr->set_learning_rate(paramConfig.learning_rate()); + if (trainerConfig_.algorithm() == "sgd") { + optimizeConfigV2.set_optimizer(paddle::OptimizerConfig::SGD); + // FIXME: config all algorithms + } else { + optimizeConfigV2.set_optimizer(paddle::OptimizerConfig::SGD); + } + std::string bytes = optimizeConfigV2.SerializeAsString(); const char *array = bytes.data(); int size = (int)bytes.size(); paddle_init_param( @@ -83,4 +99,4 @@ void NewRemoteParameterUpdater::finishBatch(real cost) { void NewRemoteParameterUpdater::startPass() {} bool NewRemoteParameterUpdater::finishPass() { return true; } -} +} // namespace paddle diff --git a/paddle/trainer/NewRemoteParameterUpdater.h b/paddle/trainer/NewRemoteParameterUpdater.h index f735185f62b3491a63e34cfc4a2ef73dae12243e..dfed00bc216b1d41bb7520619b76702f9fe650f2 100644 --- a/paddle/trainer/NewRemoteParameterUpdater.h +++ b/paddle/trainer/NewRemoteParameterUpdater.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include +#include "OptimizerConfig.pb.h" #include "ParameterUpdater.h" #include "libpaddle_pserver_cclient.h" #include "paddle/pserver/ParameterClient2.h" @@ -101,6 +102,7 @@ private: } protected: + const OptimizationConfig& trainerConfig_; /// internal parameter client object for exchanging data with pserver paddle_pserver_client parameterClient_; /// the parameters for new pserver client diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 13a1802ee3790b1255fc11f5b2053e3342c61914..0171f9d8ccd6045cb876d57684269a2a49e77f96 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -26,10 +26,17 @@ endif(WITH_GOLANG) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py) + +add_custom_command(OUTPUT ${PROJ_ROOT}/python/paddle/v2/framework/core.so + COMMAND cmake -E copy $ ${PROJ_ROOT}/python/paddle/v2/framework/core.so + DEPENDS paddle_pybind) +add_custom_target(copy_paddle_pybind ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/framework/core.so) + + add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp - DEPENDS gen_proto_py framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) + DEPENDS gen_proto_py copy_paddle_pybind framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) diff --git a/python/paddle/v2/framework/default_scope_funcs.py b/python/paddle/v2/framework/default_scope_funcs.py new file mode 100644 index 0000000000000000000000000000000000000000..4e772326c94b7ee44906c71f13e9420e078a1917 --- /dev/null +++ b/python/paddle/v2/framework/default_scope_funcs.py @@ -0,0 +1,83 @@ +""" +Default scope function. + +`Paddle` manages Scope as programming language's scope. It just a +thread-local stack of Scope. Top of that stack is current scope, the bottom +of that stack is all scopes' parent. + +Invoking `create_var/get_var` can `create/get` variable in current scope. +Invoking `enter_local_scope/leave_local_scope` can create or destroy local +scope. + +A `scoped_function` will take a `function` as input. That function will be +invoked in a new local scope. +""" + +import paddle.v2.framework.core +import threading + +__tl_scope__ = threading.local() + +__all__ = [ + 'get_cur_scope', 'enter_local_scope', 'leave_local_scope', 'create_var', + 'get_var', 'scoped_function' +] + + +def get_cur_scope(): + """ + Get current scope. + :rtype: paddle.v2.framework.core.Scope + """ + cur_scope_stack = getattr(__tl_scope__, 'cur_scope', None) + if cur_scope_stack is None: + __tl_scope__.cur_scope = list() + if len(__tl_scope__.cur_scope) == 0: + __tl_scope__.cur_scope.append(paddle.v2.framework.core.Scope(None)) + return __tl_scope__.cur_scope[-1] + + +def enter_local_scope(): + """ + Enter a new local scope + """ + cur_scope = get_cur_scope() + new_scope = paddle.v2.framework.core.Scope(cur_scope) + __tl_scope__.cur_scope.append(new_scope) + + +def leave_local_scope(): + """ + Leave local scope + """ + __tl_scope__.cur_scope.pop() + + +def create_var(name): + """ + create variable in current scope. + """ + return get_cur_scope().create_var(name) + + +def get_var(name): + """ + get variable in current scope. + """ + return get_cur_scope().get_var(name) + + +def scoped_function(func): + """ + invoke `func` in new scope. + + :param func: a callable function that will be run in new scope. + :type func: callable + """ + enter_local_scope() + try: + func() + except: + raise + finally: + leave_local_scope() diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 8cb0c5c3765a00b45177117925e320e61a1b609a..7023e82b5f08eb49fa1fee27118a7907d58312e2 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -1 +1,2 @@ -add_python_test(test_framework test_protobuf.py) +add_python_test(test_framework test_protobuf.py test_scope.py + test_default_scope_funcs.py) diff --git a/python/paddle/v2/framework/tests/test_default_scope_funcs.py b/python/paddle/v2/framework/tests/test_default_scope_funcs.py new file mode 100644 index 0000000000000000000000000000000000000000..81033deb1546c81e2566ec5474f45dc56781644a --- /dev/null +++ b/python/paddle/v2/framework/tests/test_default_scope_funcs.py @@ -0,0 +1,33 @@ +from paddle.v2.framework.default_scope_funcs import * +import unittest + + +class TestDefaultScopeFuncs(unittest.TestCase): + def test_cur_scope(self): + self.assertIsNotNone(get_cur_scope()) + + def test_none_variable(self): + self.assertIsNone(get_var("test")) + + def test_create_var_get_var(self): + var_a = create_var("var_a") + self.assertIsNotNone(var_a) + self.assertIsNotNone(get_cur_scope().get_var('var_a')) + enter_local_scope() + self.assertIsNotNone(get_cur_scope().get_var('var_a')) + leave_local_scope() + + def test_var_get_int(self): + def __new_scope__(): + i = create_var("var_i") + self.assertFalse(i.is_int()) + i.set_int(10) + self.assertTrue(i.is_int()) + self.assertEqual(10, i.get_int()) + + for _ in xrange(10): + scoped_function(__new_scope__) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_protobuf.py b/python/paddle/v2/framework/tests/test_protobuf.py index f0e60191991e2f24b0a1972afb0f6cbd3aaa4008..b8702477e64203e735bff05b115eafbb2a52172d 100644 --- a/python/paddle/v2/framework/tests/test_protobuf.py +++ b/python/paddle/v2/framework/tests/test_protobuf.py @@ -24,3 +24,7 @@ class TestFrameworkProto(unittest.TestCase): attr.type = attr_type_lib.FLOAT op_proto.type = "cos" self.assertTrue(op_proto.IsInitialized()) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/v2/framework/tests/test_scope.py b/python/paddle/v2/framework/tests/test_scope.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ee45cfc75e486c693a00d92a97ac0970195581 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_scope.py @@ -0,0 +1,37 @@ +import paddle.v2.framework.core +import unittest + + +class TestScope(unittest.TestCase): + def test_create_destroy(self): + paddle_c = paddle.v2.framework.core + scope = paddle_c.Scope(None) + self.assertIsNotNone(scope) + scope_with_parent = paddle_c.Scope(scope) + self.assertIsNotNone(scope_with_parent) + + def test_none_variable(self): + paddle_c = paddle.v2.framework.core + scope = paddle_c.Scope(None) + self.assertIsNone(scope.get_var("test")) + + def test_create_var_get_var(self): + paddle_c = paddle.v2.framework.core + scope = paddle_c.Scope(None) + var_a = scope.create_var("var_a") + self.assertIsNotNone(var_a) + self.assertIsNotNone(scope.get_var('var_a')) + scope2 = paddle_c.Scope(scope) + self.assertIsNotNone(scope2.get_var('var_a')) + + def test_var_get_int(self): + paddle_c = paddle.v2.framework.core + scope = paddle_c.Scope(None) + var = scope.create_var("test_int") + var.set_int(10) + self.assertTrue(var.is_int()) + self.assertEqual(10, var.get_int()) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/optimizer.py b/python/paddle/v2/optimizer.py index 8124e219ba499333ecdf4b34ff5352e281aaa016..390c22ee552c506fde1567efba1326a6d735ad2e 100644 --- a/python/paddle/v2/optimizer.py +++ b/python/paddle/v2/optimizer.py @@ -66,6 +66,8 @@ class Optimizer(object): if use_sparse_remote_updater: gradient_machine.prefetch(in_args) parameter_updater.getParametersRemote() + + :param pserver_spec: pserver location, eg: localhost:3000 :return: parameter_updater """ if is_local: diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index f9658a8c5df9562073c8a187074a6cb3459ac5d9..96c6c4b89a2f2e2c3ecb95213e0e0191b1998f50 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -41,6 +41,7 @@ class SGD(object): :type parameters: paddle.v2.parameters.Parameters :param extra_layers: Some layers in the neural network graph are not in the path of cost layer. + :param pserver_spec: pserver location, eg: localhost:3000 :type extra_layers: paddle.v2.config_base.Layer """ diff --git a/python/setup.py.in b/python/setup.py.in index a422b3832f4c9c60bc5406277f9ada7032f85f51..271ee6e5526981ad94710315d1472b0f4069a1aa 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -29,7 +29,9 @@ setup(name='paddle', description='Parallel Distributed Deep Learning', install_requires=setup_requires, packages=packages, - package_data={'paddle.v2.master': ['libpaddle_master.so'], }, + package_data={'paddle.v2.master': ['libpaddle_master.so'], + 'paddle.v2.framework': ['core.so'] + }, package_dir={ '': '${CMAKE_CURRENT_SOURCE_DIR}', # The paddle.v2.framework.proto will be generated while compiling.