Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into improve_pruning

.pre-commit-config.yaml

@@ -3,8 +3,8 @@
     hooks:
     -   id: remove-crlf
         files: (?!.*third_party)^.*$ | (?!.*book)^.*$
--   repo: https://github.com/reyoung/mirrors-yapf.git
-    sha: v0.13.2
+-   repo: https://github.com/PaddlePaddle/mirrors-yapf.git
+    sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37
     hooks:
     -   id: yapf
         files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$

README.md

@@ -70,7 +70,7 @@ before looking into the
 We provide [English](http://www.paddlepaddle.org/develop/doc/) and
 [Chinese](http://www.paddlepaddle.org/doc_cn/) documentation.
 
-- [Deep Learning 101](http://book.paddlepaddle.org/index.en.html)
+- [Deep Learning 101](http://book.paddlepaddle.org/index.html)
 
   You might want to start from the this online interactive book that can run in Jupyter Notebook.
 

cmake/cudnn.cmake

@@ -11,11 +11,23 @@ find_path(CUDNN_INCLUDE_DIR cudnn.h
 
 get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH)
 
+if(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR})
+    execute_process(
+        COMMAND uname -m COMMAND tr -d '\n'
+        OUTPUT_VARIABLE HOST_ARCH
+        RESULT_VARIABLE UNAME_RESULT)
+    if(${UNAME_RESULT})
+        set(HOST_ARCH "x86_64")
+    endif(${UNAME_RESULT})
+else(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR})
+    set(HOST_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR})
+endif(NOT ${CMAKE_HOST_SYSTEM_PROCESSOR})
+
 list(APPEND CUDNN_CHECK_LIBRARY_DIRS
     ${CUDNN_ROOT}
     ${CUDNN_ROOT}/lib64
     ${CUDNN_ROOT}/lib
-    ${CUDNN_ROOT}/lib/x86_64-linux-gnu
+    ${CUDNN_ROOT}/lib/${HOST_ARCH}-linux-gnu
     $ENV{CUDNN_ROOT}
     $ENV{CUDNN_ROOT}/lib64
     $ENV{CUDNN_ROOT}/lib

cmake/external/any.cmake

@@ -18,3 +18,4 @@ ExternalProject_Add(
 )
 
 add_definitions(-DANY_IMPL_ANY_CAST_MOVEABLE)
+LIST(APPEND external_project_dependencies linb_any)
\ No newline at end of file

cmake/external/gflags.cmake

@@ -26,7 +26,7 @@ ENDIF(WIN32)
 INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
 
 ExternalProject_Add(
-    gflags
+    extern_gflags
     ${EXTERNAL_PROJECT_LOG_ARGS}
     GIT_REPOSITORY  "https://github.com/gflags/gflags.git"
     PREFIX          ${GFLAGS_SOURCES_DIR}
@@ -44,4 +44,8 @@ ExternalProject_Add(
                      -DCMAKE_BUILD_TYPE:STRING=Release
 )
 
+ADD_LIBRARY(gflags STATIC IMPORTED)
+SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES})
+ADD_DEPENDENCIES(gflags extern_gflags)
+
 LIST(APPEND external_project_dependencies gflags)

cmake/external/glog.cmake

@@ -27,7 +27,7 @@ ENDIF(WIN32)
 INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR})
 
 ExternalProject_Add(
-    glog
+    extern_glog
     ${EXTERNAL_PROJECT_LOG_ARGS}
     DEPENDS gflags
     GIT_REPOSITORY  "https://github.com/google/glog.git"
@@ -48,4 +48,8 @@ ExternalProject_Add(
                      -DCMAKE_BUILD_TYPE:STRING=Release
 )
 
+ADD_LIBRARY(glog STATIC IMPORTED)
+SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES})
+ADD_DEPENDENCIES(glog extern_glog)
+
 LIST(APPEND external_project_dependencies glog)

cmake/external/gtest.cmake

@@ -35,7 +35,7 @@ IF(WITH_TESTING)
     ENDIF(WIN32)
 
     ExternalProject_Add(
-        gtest
+        extern_gtest
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY  "https://github.com/google/googletest.git"
         GIT_TAG         "release-1.8.0"
@@ -55,5 +55,14 @@ IF(WITH_TESTING)
                          -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
                          -DCMAKE_BUILD_TYPE:STRING=Release
     )
-    LIST(APPEND external_project_dependencies gtest)
+
+    ADD_LIBRARY(gtest STATIC IMPORTED)
+    SET_PROPERTY(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES})
+    ADD_DEPENDENCIES(gtest extern_gtest)
+
+    ADD_LIBRARY(gtest_main STATIC IMPORTED)
+    SET_PROPERTY(TARGET gtest_main PROPERTY IMPORTED_LOCATION ${GTEST_MAIN_LIBRARIES})
+    ADD_DEPENDENCIES(gtest_main extern_gtest)
+
+    LIST(APPEND external_project_dependencies gtest gtest_main)
 ENDIF(WITH_TESTING)

cmake/external/openblas.cmake

@@ -37,11 +37,11 @@ IF(NOT ${CBLAS_FOUND})
         SET(OPTIONAL_ARGS HOSTCC=${HOST_C_COMPILER} TARGET=ARMV7 USE_THREAD=0 libs)
     ELSE()
         SET(OPENBLAS_COMMIT "v0.2.19")
-        SET(OPENBLAS_ARGS DYNAMIC_ARCH=1 libs)
+        SET(OPTIONAL_ARGS DYNAMIC_ARCH=1 libs NUM_THREADS=64)
     ENDIF()
 
     ExternalProject_Add(
-        openblas
+        extern_openblas
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY      https://github.com/xianyi/OpenBLAS.git
         GIT_TAG             ${OPENBLAS_COMMIT}
@@ -53,8 +53,14 @@ IF(NOT ${CBLAS_FOUND})
         UPDATE_COMMAND      ""
         CONFIGURE_COMMAND   ""
     )
-    LIST(APPEND external_project_dependencies openblas)
 ENDIF(NOT ${CBLAS_FOUND})
 
 MESSAGE(STATUS "BLAS library: ${CBLAS_LIBRARIES}")
 INCLUDE_DIRECTORIES(${CBLAS_INC_DIR})
+
+ADD_LIBRARY(cblas STATIC IMPORTED)
+SET_PROPERTY(TARGET cblas PROPERTY IMPORTED_LOCATION ${CBLAS_LIBRARIES})
+IF(NOT ${CBLAS_FOUND})
+    ADD_DEPENDENCIES(cblas extern_openblas)
+    LIST(APPEND external_project_dependencies cblas)
+ENDIF(NOT ${CBLAS_FOUND})

cmake/external/protobuf.cmake

@@ -17,9 +17,14 @@ INCLUDE(ExternalProject)
 macro(PROMPT_PROTOBUF_LIB)
     MESSAGE(STATUS "Protobuf protoc executable: ${PROTOBUF_PROTOC_EXECUTABLE}")
     MESSAGE(STATUS "Protobuf library: ${PROTOBUF_LIBRARY}")
+    MESSAGE(STATUS "Protobuf version: ${PROTOBUF_VERSION}")
     INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR})
     RETURN()
 endmacro()
+macro(SET_PROTOBUF_VERSION)
+    EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION)
+    STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}")
+endmacro()
 
 set(PROTOBUF_ROOT "" CACHE PATH "Folder contains protobuf")
 if (NOT "${PROTOBUF_ROOT}" STREQUAL "")
@@ -30,6 +35,7 @@ if (NOT "${PROTOBUF_ROOT}" STREQUAL "")
     find_program(PROTOBUF_PROTOC_EXECUTABLE protoc PATHS ${PROTOBUF_ROOT}/bin)
     if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOBUF_LITE_LIBRARY AND PROTOBUF_PROTOC_LIBRARY AND PROTOBUF_PROTOC_EXECUTABLE)
         message(STATUS "Using custom protobuf library in ${PROTOBUF_ROOT}.")
+        SET_PROTOBUF_VERSION()
         PROMPT_PROTOBUF_LIB()
     else()
         message(WARNING "Cannot find protobuf library in ${PROTOBUF_ROOT}.")
@@ -100,8 +106,7 @@ IF(NOT CMAKE_CROSSCOMPILING)
     FIND_PACKAGE(Protobuf ${PROTOBUF_VERSION})
 
     IF(PROTOBUF_FOUND)
-        EXEC_PROGRAM(${PROTOBUF_PROTOC_EXECUTABLE} ARGS --version OUTPUT_VARIABLE PROTOBUF_VERSION)
-        STRING(REGEX MATCH "[0-9]+.[0-9]+" PROTOBUF_VERSION "${PROTOBUF_VERSION}")
+        SET_PROTOBUF_VERSION()
         IF("${PROTOBUF_VERSION}" VERSION_LESS "3.1.0")
             SET(PROTOBUF_FOUND OFF)
         ENDIF()

cmake/external/warpctc.cmake

@@ -43,7 +43,7 @@ ELSE()
 ENDIF()
 
 ExternalProject_Add(
-    warpctc
+    extern_warpctc
     ${EXTERNAL_PROJECT_LOG_ARGS}
     GIT_REPOSITORY  "https://github.com/gangliao/warp-ctc.git"
     PREFIX          ${WARPCTC_SOURCES_DIR}
@@ -65,4 +65,8 @@ ExternalProject_Add(
                      -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR}
 )
 
+ADD_LIBRARY(warpctc STATIC IMPORTED)
+SET_PROPERTY(TARGET warpctc PROPERTY IMPORTED_LOCATION ${WARPCTC_LIBRARIES})
+ADD_DEPENDENCIES(warpctc extern_warpctc)
+
 LIST(APPEND external_project_dependencies warpctc)

cmake/generic.cmake

 # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,7 +16,7 @@
 # To simplify the build process of PaddlePaddle, we defined couple of
 # fundamental abstractions, e.g., how to build library, binary and
 # test in C++, CUDA and Go.
-# 
+#
 # -------------------------------------------
 #    C++	      CUDA C++	      Go
 # -------------------------------------------
@@ -29,6 +29,11 @@
 # https://cmake.org/cmake/help/v3.0/module/CMakeParseArguments.html
 #
 
+if(NOT APPLE)
+    find_package(Threads REQUIRED)
+    link_libraries(${CMAKE_THREAD_LIBS_INIT})
+endif(NOT APPLE)
+
 # cc_library parses tensor.cc and figures out that target also depend on tensor.h.
 # cc_library(tensor
 #   SRCS
@@ -45,7 +50,9 @@ function(cc_library TARGET_NAME)
   else()
     add_library(${TARGET_NAME} STATIC ${cc_library_SRCS})
   endif()
-  add_dependencies(${TARGET_NAME} ${cc_library_DEPS} ${external_project_dependencies})
+  if (cc_library_DEPS)
+    add_dependencies(${TARGET_NAME} ${cc_library_DEPS})
+  endif()
 endfunction(cc_library)
 
 # cc_binary parses tensor.cc and figures out that target also depend on tensor.h.
@@ -58,8 +65,7 @@ function(cc_binary TARGET_NAME)
   set(multiValueArgs SRCS DEPS)
   cmake_parse_arguments(cc_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
   add_executable(${TARGET_NAME} ${cc_binary_SRCS})
-  link_paddle_exe(${TARGET_NAME})
-  if(cc_binary_DEPS)  
+  if(cc_binary_DEPS)
     target_link_libraries(${TARGET_NAME} ${cc_binary_DEPS})
     add_dependencies(${TARGET_NAME} ${cc_binary_DEPS})
   endif()
@@ -73,17 +79,16 @@ endfunction(cc_binary)
 #   DEPS
 #   tensor)
 function(cc_test TARGET_NAME)
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs SRCS DEPS)
-  cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-  add_executable(${TARGET_NAME} ${cc_test_SRCS})
-  link_paddle_test(${TARGET_NAME})
-  if(cc_test_DEPS)
-    target_link_libraries(${TARGET_NAME} ${cc_test_DEPS})
-    add_dependencies(${TARGET_NAME} ${cc_test_DEPS})
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    cmake_parse_arguments(cc_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    add_executable(${TARGET_NAME} ${cc_test_SRCS})
+    target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main)
+    add_dependencies(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main)
+    add_test(${TARGET_NAME} ${TARGET_NAME})
   endif()
-  add_test(${TARGET_NAME} ${TARGET_NAME})
 endfunction(cc_test)
 
 # Suppose that ops.cu includes global functions that take Tensor as
@@ -95,28 +100,33 @@ endfunction(cc_test)
 #   DEPS
 #   tensor)
 function(nv_library TARGET_NAME)
-  set(options OPTIONAL)
-  set(oneValueArgs "")
-  set(multiValueArgs SRCS DEPS)
-  cmake_parse_arguments(nv_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-  if (${nv_library_OPTIONAL} STREQUAL "SHARED")
-    cuda_add_library(${TARGET_NAME} SHARED ${nv_library_SRCS})
-  else()
-    cuda_add_library(${TARGET_NAME} STATIC ${nv_library_SRCS})
+  if (WITH_GPU)
+    set(options OPTIONAL)
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    cmake_parse_arguments(nv_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    if (${nv_library_OPTIONAL} STREQUAL "SHARED")
+      cuda_add_library(${TARGET_NAME} SHARED ${nv_library_SRCS})
+    else()
+      cuda_add_library(${TARGET_NAME} STATIC ${nv_library_SRCS})
+    endif()
+    if (nv_library_DEPS)
+      add_dependencies(${TARGET_NAME} ${nv_library_DEPS})
+    endif()
   endif()
-  add_dependencies(${TARGET_NAME} ${nv_library_DEPS} ${external_project_dependencies})
 endfunction(nv_library)
 
 function(nv_binary TARGET_NAME)
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs SRCS DEPS)
-  cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-  cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS})
-  link_paddle_exe(${TARGET_NAME})  
-  if(nv_binary_DEPS)
-    target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS})
-    add_dependencies(${TARGET_NAME} ${nv_binary_DEPS})
+  if (WITH_GPU)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    cmake_parse_arguments(nv_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    cuda_add_executable(${TARGET_NAME} ${nv_binary_SRCS})
+    if(nv_binary_DEPS)
+      target_link_libraries(${TARGET_NAME} ${nv_binary_DEPS})
+      add_dependencies(${TARGET_NAME} ${nv_binary_DEPS})
+    endif()
   endif()
 endfunction(nv_binary)
 
@@ -128,17 +138,16 @@ endfunction(nv_binary)
 #   DEPS
 #   ops)
 function(nv_test TARGET_NAME)
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs SRCS DEPS)
-  cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-  cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS})
-  link_paddle_test(${TARGET_NAME})  
-  if(nv_test_DEPS)
-    target_link_libraries(${TARGET_NAME} ${nv_test_DEPS})
-    add_dependencies(${TARGET_NAME} ${nv_test_DEPS})
+  if (WITH_GPU AND WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    cmake_parse_arguments(nv_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    cuda_add_executable(${TARGET_NAME} ${nv_test_SRCS})
+    target_link_libraries(${TARGET_NAME} ${nv_test_DEPS} gtest gtest_main)
+    add_dependencies(${TARGET_NAME} ${nv_test_DEPS} gtest gtest_main)
+    add_test(${TARGET_NAME} ${TARGET_NAME})
   endif()
-  add_test(${TARGET_NAME} ${TARGET_NAME})
 endfunction(nv_test)
 
 set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
@@ -164,7 +173,7 @@ function(go_library TARGET_NAME)
       set(LIB_NAME "lib${TARGET_NAME}.dylib")
     else()
       set(LIB_NAME "lib${TARGET_NAME}.so")
-    endif()  
+    endif()
   else()
     set(BUILD_MODE "-buildmode=c-archive")
     set(LIB_NAME "lib${TARGET_NAME}.a")
@@ -190,8 +199,8 @@ function(go_binary TARGET_NAME)
     COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
     -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
     ${go_library_SRCS}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})  
-  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS})  
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS})
   install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin)
 endfunction(go_binary)
 
@@ -204,8 +213,8 @@ function(go_test TARGET_NAME)
     COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} test
     -c -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
     ${go_test_SRCS}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})  
-  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS})  
+    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
+  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS})
   add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME})
 endfunction(go_test)
 

doc/design/cluster_train/pserver_client.md

@@ -136,6 +136,9 @@ int paddle_send_grads(paddle_pserver_client* client, const paddle_gradient* grad
 /**
  * @brief paddle_get_params gets parameters from parameter servers.
  *
+ * paddle_get_params will block until parameters are initialized on
+ * the parameter servers.
+ *
  * @param names the array of names of the parameters to get.
  * @param dst the destination array of parameters to save to.
  * @param len the length of the names array and the paddle_parameter

doc/howto/deep_model/rnn/hierarchical_layer_cn.rst

@@ -28,17 +28,17 @@ pooling 的使用示例如下，详细见 :ref:`api_v2.layer_pooling` 配置API
 
         seq_pool = pooling(input=layer,
                            pooling_type=pooling.Max(),
-                           agg_level=AggregateLevel.EACH_SEQUENCE)
+                           agg_level=AggregateLevel.TO_SEQUENCE)
         
 - `pooling_type` 目前支持两种，分别是：pooling.Max()和pooling.Avg()。
 
-- `agg_level=AggregateLevel.EACH_TIMESTEP` 时（默认值）：
+- `agg_level=AggregateLevel.TO_NO_SEQUENCE` 时（默认值）：
 
   - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列
   - 输入：一个双层序列，或一个单层序列
   - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值）
 
-- `agg_level=AggregateLevel.EACH_SEQUENCE` 时：
+- `agg_level=AggregateLevel.TO_SEQUENCE` 时：
 
   - 作用：一个双层序列经过运算变成一个单层序列
   - 输入：必须是一个双层序列
@@ -52,15 +52,15 @@ last_seq 的使用示例如下（ :ref:`api_v2.layer_first_seq` 类似），详
 ..	code-block:: bash
 
         last = last_seq(input=layer,
-                        agg_level=AggregateLevel.EACH_SEQUENCE)
+                        agg_level=AggregateLevel.TO_SEQUENCE)
         
-- `agg_level=AggregateLevel.EACH_TIMESTEP` 时（默认值）：
+- `agg_level=AggregateLevel.TO_NO_SEQUENCE` 时（默认值）：
 
   - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列
   - 输入：一个双层序列或一个单层序列
   - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。
 
-- `agg_level=AggregateLevel.EACH_SEQUENCE` 时：
+- `agg_level=AggregateLevel.TO_SEQUENCE` 时：
   - 作用：一个双层序列经过运算变成一个单层序列
   - 输入：必须是一个双层序列
   - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。
@@ -74,9 +74,9 @@ expand 的使用示例如下，详细见 :ref:`api_v2.layer_expand` 配置API。
 
         ex = expand(input=layer1,
                     expand_as=layer2,
-                    expand_level=ExpandLevel.FROM_TIMESTEP)
+                    expand_level=ExpandLevel.FROM_NO_SEQUENCE)
         
-- `expand_level=ExpandLevel.FROM_TIMESTEP` 时（默认值）：
+- `expand_level=ExpandLevel.FROM_NO_SEQUENCE` 时（默认值）：
 
   - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列
   - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2 可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息

doc/howto/deep_model/rnn/hrnn_rnn_api_compare_cn.rst

@@ -81,7 +81,7 @@
   
   * 在本例中，我们将原始数据的每一组，通过\ :code:`recurrent_group`\ 进行拆解，拆解成的每一句话再通过一个LSTM网络。这和单层RNN的配置是等价的。
 
-* 与单层RNN的配置类似，我们只需要使用LSTM encode成的最后一个向量。所以对\ :code:`recurrent_group`\ 进行了\ :code:`last_seq`\ 操作。但和单层RNN不同，我们是对每一个子序列取最后一个元素，因此\ :code:`agg_level=AggregateLevel.EACH_SEQUENCE`\ 。
+* 与单层RNN的配置类似，我们只需要使用LSTM encode成的最后一个向量。所以对\ :code:`recurrent_group`\ 进行了\ :code:`last_seq`\ 操作。但和单层RNN不同，我们是对每一个子序列取最后一个元素，因此\ :code:`agg_level=AggregateLevel.TO_SEQUENCE`\ 。
 
 * 至此，\ :code:`lstm_last`\ 便和单层RNN配置中的\ :code:`lstm_last`\ 具有相同的结果了。
 

paddle/go/cclient/cmake/CMakeDetermineGoCompiler.cmake → go/cmake/CMakeDetermineGoCompiler.cmake

@@ -38,7 +38,7 @@ endif()
 
 mark_as_advanced(CMAKE_Go_COMPILER)
 
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CMakeGoCompiler.cmake.in
+configure_file(${CMAKE_MODULE_PATH}/CMakeGoCompiler.cmake.in
   ${CMAKE_PLATFORM_INFO_DIR}/CMakeGoCompiler.cmake @ONLY)
 
 set(CMAKE_Go_COMPILER_ENV_VAR "GO_COMPILER")

paddle/go/cclient/cmake/flags.cmake → go/cmake/flags.cmake

@@ -21,7 +21,7 @@ function(CheckCompilerCXX11Flag)
             if (${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.3)
                 message(FATAL_ERROR "Unsupported Clang version. Clang >= 3.3 required.")
             endif()
-        endif()   
+        endif()
     endif()
 endfunction()
 
@@ -42,4 +42,4 @@ if (CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0")
       list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60")
 endif()
 
-set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS})
+set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS})
\ No newline at end of file

paddle/go/cclient/cmake/golang.cmake → go/cmake/golang.cmake

 set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
 file(MAKE_DIRECTORY ${GOPATH})
+set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle")
+file(MAKE_DIRECTORY ${PADDLE_IN_GOPATH})
 
-function(ExternalGoProject_Add TARG)
-  add_custom_target(${TARG} env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get ${ARGN})
-endfunction(ExternalGoProject_Add)
-
-function(add_go_executable NAME)
-  file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
-  add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp 
-    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
-    -o "${CMAKE_CURRENT_BINARY_DIR}/${NAME}"
-    ${CMAKE_GO_FLAGS} ${GO_SOURCE}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
-
-  add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN})
-  install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${NAME} DESTINATION bin)
-endfunction(add_go_executable)
-
-
-function(ADD_GO_LIBRARY NAME BUILD_TYPE)
+function(GO_LIBRARY NAME BUILD_TYPE)
   if(BUILD_TYPE STREQUAL "STATIC")
     set(BUILD_MODE -buildmode=c-archive)
     set(LIB_NAME "lib${NAME}.a")
@@ -32,6 +17,24 @@ function(ADD_GO_LIBRARY NAME BUILD_TYPE)
   endif()
 
   file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
+  file(RELATIVE_PATH rel ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
+
+  # find Paddle directory.
+  get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
+  get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
+  get_filename_component(PADDLE_DIR ${PARENT_DIR} DIRECTORY)
+
+  # automatically get all dependencies specified in the source code
+  # for given target.
+  add_custom_target(goGet env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ${rel}/...)
+
+  # make a symlink that references Paddle inside $GOPATH, so go get
+  # will use the local changes in Paddle rather than checkout Paddle
+  # in github.
+  add_custom_target(copyPaddle
+    COMMAND ln -sf ${PADDLE_DIR} ${PADDLE_IN_GOPATH})
+  add_dependencies(goGet copyPaddle)
+
   add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
     COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
     -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
@@ -39,8 +42,9 @@ function(ADD_GO_LIBRARY NAME BUILD_TYPE)
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
 
   add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN})
+  add_dependencies(${NAME} goGet)
 
   if(NOT BUILD_TYPE STREQUAL "STATIC")
     install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION bin)
   endif()
-endfunction(ADD_GO_LIBRARY)
+endfunction(GO_LIBRARY)

paddle/go/cmd/master/master.go → go/cmd/master/master.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/namsral/flag"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/master"
-	"github.com/PaddlePaddle/Paddle/paddle/go/recordio"
+	"github.com/PaddlePaddle/Paddle/go/master"
+	"github.com/PaddlePaddle/Paddle/go/recordio"
 )
 
 func main() {

paddle/go/cmd/pserver/pserver.go → go/cmd/pserver/pserver.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/namsral/flag"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/pserver"
+	"github.com/PaddlePaddle/Paddle/go/pserver"
 )
 
 func main() {

paddle/go/master/service.go → go/master/service.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/recordio"
+	"github.com/PaddlePaddle/Paddle/go/recordio"
 )
 
 const (

go/pserver/cclient/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+
+get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
+get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
+
+project(cxx_go C Go)
+
+include(golang)
+include(flags)
+
+go_library(client STATIC)
+add_subdirectory(test)

paddle/go/cclient/cclient.go → go/pserver/cclient/cclient.go

@@ -39,10 +39,11 @@ import "C"
 
 import (
 	"log"
+	"strings"
 	"sync"
 	"unsafe"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/pserver"
+	"github.com/PaddlePaddle/Paddle/go/pserver"
 )
 
 var nullPtr = unsafe.Pointer(uintptr(0))
@@ -78,34 +79,54 @@ func cArrayToSlice(p unsafe.Pointer, len int) []byte {
 		return nil
 	}
 
-	// create a Go clice backed by a C array,
-	// reference: https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	// create a Go clice backed by a C array, reference:
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	//
+	// Go garbage collector will not interact with this data, need
+	// to be freed properly.
 	return (*[1 << 30]byte)(p)[:len:len]
 }
 
+type selector bool
+
+func (s selector) Select() bool {
+	return bool(s)
+}
+
+type lister []pserver.Server
+
+func (l lister) List() []pserver.Server {
+	return l
+}
+
 //export paddle_new_pserver_client
-func paddle_new_pserver_client(addr *C.char) C.client {
-	c := pserver.NewClient(C.GoString(addr))
+func paddle_new_pserver_client(addrs *C.char, selected int) C.client {
+	a := C.GoString(addrs)
+	as := strings.Split(a, ",")
+	servers := make([]pserver.Server, len(as))
+	for i := range as {
+		servers[i].Index = i
+		servers[i].Addr = as[i]
+	}
+	c := pserver.NewClient(lister(servers), len(as), selector(selected != 0))
 	return add(c)
 }
 
+//export paddle_new_etcd_pserver_client
+func paddle_new_etcd_pserver_client(etcd_addr *C.char) C.client {
+	// TODO(helin): fault tolerant pserver client using etcd.
+	panic("not implemented.")
+}
+
 //export paddle_pserver_client_release
 func paddle_pserver_client_release(client C.client) {
-	c := remove(client)
-	c.Cleanup()
+	remove(client)
 }
 
 //export paddle_begin_init_params
-func paddle_begin_init_params(client C.client, pserver_config unsafe.Pointer, config_len C.int) C.int {
+func paddle_begin_init_params(client C.client) C.int {
 	c := get(client)
-	b := cArrayToSlice(pserver_config, int(config_len))
-	selected, err := c.BeginInitParams(b)
-	if err != nil {
-		log.Println(err)
-		return -1
-	}
-
-	if selected {
+	if selected := c.BeginInitParams(); selected {
 		return 1
 	}
 	return 0
@@ -227,7 +248,7 @@ func paddle_get_params(client C.client, names **C.char, dst **C.paddle_parameter
 func paddle_save_model(client C.client, path *C.char) C.int {
 	p := C.GoString(path)
 	c := get(client)
-	err := c.SaveModel(p)
+	err := c.Save(p)
 	if err != nil {
 		log.Println(err)
 		return -1

paddle/go/cclient/test/CMakeLists.txt → go/pserver/cclient/test/CMakeLists.txt

@@ -4,5 +4,8 @@ include_directories(${CMAKE_BINARY_DIR})
 
 add_executable(main main.c)
 add_dependencies(main client)
-set (CMAKE_EXE_LINKER_FLAGS "-pthread")
+
+if(APPLE)
+  set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security")
+endif()
 target_link_libraries(main ${CMAKE_BINARY_DIR}/libclient.a)

paddle/go/cclient/test/main.c → go/pserver/cclient/test/main.c

-#include "libclient.h"
+#include <stdio.h>
 
-//#include "gtest/gtest.h"
+#include "libclient.h"
 
-void panic() {
+void fail() {
   // TODO(helin): fix: gtest using cmake is not working, using this
   // hacky way for now.
-  *(void*)0;
+  printf("test failed.\n");
+  exit(-1);
 }
 
 int main() {
   char addr[] = "localhost:3000";
-  client c = paddle_new_pserver_client(addr);
+  client c = paddle_new_pserver_client(addr, 1);
 retry:
-  if (paddle_begin_init_params(c, NULL, 0)) {
+  if (paddle_begin_init_params(c)) {
     paddle_parameter param;
     char name_a[] = "param_a";
     char name_b[] = "param_b";
-    char content[] = {0x00, 0x11, 0x22};
+    unsigned char content[] = {0x00, 0x11, 0x22};
     param.element_type = PADDLE_ELEMENT_TYPE_FLOAT32;
     param.name = name_a;
     param.content = content;
@@ -35,34 +36,34 @@ retry:
       goto retry;
     }
   } else {
-    panic();
+    fail();
   }
 
-  char content[] = {0x00, 0x11, 0x22};
+  unsigned char content[] = {0x00, 0x11, 0x22};
   paddle_gradient grads[2] = {
       {"param_a", PADDLE_ELEMENT_TYPE_INT32, content, 3},
       {"param_b", PADDLE_ELEMENT_TYPE_FLOAT32, content, 3}};
 
   if (!paddle_send_grads(c, grads, 2)) {
-    panic();
+    fail();
   }
 
   paddle_parameter* params[2] = {NULL, NULL};
   char* names[] = {"param_a", "param_b"};
   if (!paddle_get_params(c, names, params, 2)) {
-    panic();
+    fail();
   }
 
   // get parameters again by reusing the allocated parameter buffers.
   if (!paddle_get_params(c, names, params, 2)) {
-    panic();
+    fail();
   }
 
   paddle_release_param(params[0]);
   paddle_release_param(params[1]);
 
   if (!paddle_save_model(c, "/tmp/")) {
-    panic();
+    fail();
   }
 
   return 0;

go/pserver/client.go

+package pserver
+
+import (
+	"hash/fnv"
+	"log"
+	"sort"
+	"time"
+
+	"github.com/PaddlePaddle/Paddle/go/pserver/internal/connection"
+)
+
+// TODO(helin): add RPC call retry logic
+
+// Selector selects if the client should initialize parameter servers.
+type Selector interface {
+	Select() bool
+}
+
+// Server is the identification of a parameter Server.
+type Server struct {
+	Index int
+	Addr  string
+}
+
+// Lister lists currently available parameter servers.
+type Lister interface {
+	List() []Server
+}
+
+// Client is the client to parameter servers.
+type Client struct {
+	sel      Selector
+	pservers []*connection.Conn
+}
+
+// NewClient creates a new client.
+func NewClient(l Lister, pserverNum int, sel Selector) *Client {
+	c := &Client{sel: sel}
+	c.pservers = make([]*connection.Conn, pserverNum)
+	for i := 0; i < pserverNum; i++ {
+		c.pservers[i] = connection.New()
+	}
+	go c.monitorPservers(l, pserverNum)
+	return c
+}
+
+// monitorPservers monitors pserver addresses, and updates connection
+// when the address changes.
+func (c *Client) monitorPservers(l Lister, pserverNum int) {
+	knownServers := make([]Server, pserverNum)
+	ticker := time.NewTicker(10 * time.Second)
+	monitor := func() {
+		curServers := make([]Server, pserverNum)
+		list := l.List()
+		for _, l := range list {
+			curServers[l.Index] = l
+		}
+
+		for i := range knownServers {
+			if knownServers[i].Addr != curServers[i].Addr {
+				err := c.pservers[i].Connect(curServers[i].Addr)
+				if err != nil {
+					log.Println(err)
+
+					// connect to addr failed, set
+					// to last known addr in order
+					// to retry next time.
+					curServers[i].Addr = knownServers[i].Addr
+				}
+			}
+		}
+
+		knownServers = curServers
+	}
+
+	monitor()
+	for _ = range ticker.C {
+		monitor()
+	}
+}
+
+// BeginInitParams begins to initialize parameters on parameter
+// servers.
+//
+// BeginInitParams will be called from multiple trainers, only one
+// trainer will be selected to initialize the parameters on parameter
+// servers. Other trainers will be blocked until the initialization is
+// done, and they need to get the initialized parameters from
+// parameter servers using GetParams.
+func (c *Client) BeginInitParams() bool {
+	return c.sel.Select()
+}
+
+// InitParam initializes the parameter on parameter servers.
+func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error {
+	var dummy int
+	return c.pservers[c.partition(paramWithConfigs.Param.Name)].Call("Service.InitParam", paramWithConfigs, &dummy)
+}
+
+// FinishInitParams tells parameter servers client has sent all
+// parameters to parameter servers as initialization.
+func (c *Client) FinishInitParams() error {
+	for _, p := range c.pservers {
+		var dummy int
+		err := p.Call("Service.FinishInitParams", dummy, &dummy)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SendGrads sends gradients to parameter servers for updating
+// parameters.
+func (c *Client) SendGrads(grads []Gradient) error {
+	errCh := make(chan error, len(grads))
+	for _, g := range grads {
+		go func(g Gradient) {
+			var dummy int
+			err := c.pservers[c.partition(g.Name)].Call("Service.SendGrad", g, &dummy)
+			errCh <- err
+		}(g)
+	}
+
+	recv := 0
+	for err := range errCh {
+		if err != nil {
+			return err
+		}
+
+		recv++
+		if recv == len(grads) {
+			break
+		}
+	}
+	return nil
+}
+
+type result struct {
+	idx   int
+	param Parameter
+	err   error
+}
+
+type results []result
+
+func (r results) Len() int {
+	return len(r)
+}
+
+func (r results) Less(i int, j int) bool {
+	return r[i].idx < r[j].idx
+}
+
+func (r results) Swap(i int, j int) {
+	r[i], r[j] = r[j], r[i]
+}
+
+// GetParams gets parameters from parameter servers.
+func (c *Client) GetParams(names []string) ([]Parameter, error) {
+	rCh := make(chan result, len(names))
+
+	for idx, name := range names {
+		go func(name string, idx int) {
+			var parameter Parameter
+			err := c.pservers[c.partition(name)].Call("Service.GetParam", name, &parameter)
+			rCh <- result{idx: idx, param: parameter, err: err}
+		}(name, idx)
+	}
+
+	var rs results
+	recv := 0
+	for r := range rCh {
+		if r.err != nil {
+			return nil, r.err
+		}
+		rs = append(rs, r)
+
+		recv++
+		if recv == len(names) {
+			break
+		}
+	}
+	sort.Sort(rs)
+
+	ps := make([]Parameter, len(rs))
+	for i := range rs {
+		ps[i] = rs[i].param
+	}
+
+	return ps, nil
+}
+
+// Save indicates parameters to save the parameter to the given path.
+func (c *Client) Save(path string) error {
+	errCh := make(chan error, len(c.pservers))
+
+	for _, p := range c.pservers {
+		var dummy int
+		err := p.Call("Service.Save", path, &dummy)
+		errCh <- err
+	}
+
+	recv := 0
+	for err := range errCh {
+		if err != nil {
+			return err
+		}
+
+		recv++
+		if recv == len(c.pservers) {
+			break
+		}
+	}
+
+	// TODO(helin): there will be many files under path, need to
+	// merge them into a single file.
+	return nil
+}
+
+func strHash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// TODO(helin): now partition only select which parameter server to
+// send the entire parameter. We need to partition a parameter into
+// small blocks and send to different parameter servers.
+func (c *Client) partition(key string) int {
+	return int(strHash(key) % uint32(len(c.pservers)))
+}

go/pserver/client_test.go

+package pserver_test
+
+import (
+	"net"
+	"net/http"
+	"net/rpc"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/PaddlePaddle/Paddle/go/pserver"
+)
+
+const numPserver = 10
+
+var port [numPserver]int
+
+func init() {
+	for i := 0; i < numPserver; i++ {
+		l, err := net.Listen("tcp", ":0")
+		if err != nil {
+			panic(err)
+		}
+
+		ss := strings.Split(l.Addr().String(), ":")
+		p, err := strconv.Atoi(ss[len(ss)-1])
+		if err != nil {
+			panic(err)
+		}
+		port[i] = p
+
+		go func(l net.Listener) {
+			s := pserver.NewService()
+			server := rpc.NewServer()
+			err := server.Register(s)
+			if err != nil {
+				panic(err)
+			}
+
+			mux := http.NewServeMux()
+			mux.Handle(rpc.DefaultRPCPath, server)
+			err = http.Serve(l, mux)
+			if err != nil {
+				panic(err)
+			}
+		}(l)
+	}
+}
+
+type selector bool
+
+func (s selector) Select() bool {
+	return bool(s)
+}
+
+type lister []pserver.Server
+
+func (l lister) List() []pserver.Server {
+	return l
+}
+
+func TestClientFull(t *testing.T) {
+	servers := make([]pserver.Server, numPserver)
+	for i := 0; i < numPserver; i++ {
+		servers[i] = pserver.Server{Index: i, Addr: ":" + strconv.Itoa(port[i])}
+	}
+	c := pserver.NewClient(lister(servers), len(servers), selector(true))
+	selected := c.BeginInitParams()
+	if !selected {
+		t.Fatal("should be selected.")
+	}
+
+	const numParameter = 100
+	for i := 0; i < numParameter; i++ {
+		var p pserver.Parameter
+		p.Name = "p_" + strconv.Itoa(i)
+		p.ElementType = pserver.Float32
+		p.Content = make([]byte, (i+1)*100)
+		err := c.InitParam(pserver.ParameterWithConfig{Param: p})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err := c.FinishInitParams()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var grads []pserver.Gradient
+	for i := 0; i < numParameter/2; i++ {
+		var g pserver.Gradient
+		g.Name = "p_" + strconv.Itoa(i)
+		g.ElementType = pserver.Float32
+		g.Content = make([]byte, (i+1)*100)
+		grads = append(grads, g)
+	}
+
+	err = c.SendGrads(grads)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	names := make([]string, numParameter)
+	for i := 0; i < numParameter; i++ {
+		names[i] = "p_" + strconv.Itoa(i)
+	}
+
+	params, err := c.GetParams(names)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(names) != len(params) {
+		t.Fatalf("parameter size not match, need: %d, have: %d", len(names), len(params))
+	}
+
+	for i := range params {
+		if names[i] != params[i].Name {
+			t.Fatalf("order of returned parameter does not required: parameter name: %s, required name: %s", names[i], params[i])
+		}
+	}
+}

go/pserver/internal/connection/conn.go

+package connection
+
+import (
+	"errors"
+	"net/rpc"
+	"sync"
+)
+
+// TODO(helin): add TCP re-connect logic
+
+// Conn is a connection to a parameter server
+type Conn struct {
+	mu       sync.Mutex
+	client   *rpc.Client
+	waitConn chan struct{}
+}
+
+// New creates a new connection.
+func New() *Conn {
+	c := &Conn{}
+	return c
+}
+
+// Connect connects the connection to a address.
+func (c *Conn) Connect(addr string) error {
+	c.mu.Lock()
+	if c.client != nil {
+		err := c.client.Close()
+		if err != nil {
+			c.mu.Unlock()
+			return err
+		}
+
+		c.client = nil
+	}
+	c.mu.Unlock()
+
+	client, err := rpc.DialHTTP("tcp", addr)
+	if err != nil {
+		return err
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.client == nil {
+		c.client = client
+		if c.waitConn != nil {
+			close(c.waitConn)
+			c.waitConn = nil
+		}
+	} else {
+		return errors.New("client already set from a concurrent goroutine")
+	}
+
+	return nil
+}
+
+// Call make a RPC call.
+//
+// Call will be blocked until the connection to remote RPC service
+// being established.
+func (c *Conn) Call(serviceMethod string, args interface{}, reply interface{}) error {
+	c.mu.Lock()
+	client := c.client
+	var waitCh chan struct{}
+	if client == nil {
+		if c.waitConn != nil {
+			waitCh = c.waitConn
+		} else {
+			waitCh = make(chan struct{})
+			c.waitConn = waitCh
+		}
+	}
+	c.mu.Unlock()
+
+	if waitCh != nil {
+		// wait until new connection being established
+		<-waitCh
+		return c.Call(serviceMethod, args, reply)
+	}
+
+	return client.Call(serviceMethod, args, reply)
+}

paddle/go/pserver/optimizer.go → go/pserver/optimizer.go

@@ -29,11 +29,11 @@ func newOptimizer(t optimizerType, learning_rate float64) *optimizer {
 
 func (o *optimizer) UpdateParameter(p Parameter, g Gradient) error {
 	if len(p.Content) != len(g.Content) {
-		return fmt.Errorf("parameter and gradient length not match, parameter: %d, gradient: %d", len(p.Content), len(g.Content))
+		return fmt.Errorf("Name: %s, parameter and gradient length not match, parameter: %d, gradient: %d", p.Name, len(p.Content), len(g.Content))
 	}
 
 	if p.ElementType != g.ElementType {
-		return fmt.Errorf("parameter and gradient element type not match, parameter: %v, gradient: %v", p.ElementType, g.ElementType)
+		return fmt.Errorf("Name: %s, parameter and gradient element type not match, parameter: %v, gradient: %v", p.Name, p.ElementType, g.ElementType)
 	}
 
 	r := C.paddle_update_parameter(o.opt, unsafe.Pointer(&p.Content[0]), C.paddle_element_type(p.ElementType), unsafe.Pointer(&g.Content[0]), C.int(len(g.Content)))

paddle/go/pserver/service.go → go/pserver/service.go

@@ -49,33 +49,12 @@ type Service struct {
 
 // NewService creates a new service.
 func NewService() *Service {
-	s := &Service{}
+	s := &Service{opt: newOptimizer(sgd, 0.01)}
 	s.paramMap = make(map[string]Parameter)
 	s.initialized = make(chan struct{})
 	return s
 }
 
-// BeginInitParams tells the parameter server that the parameter
-// initialization has begun.
-func (s *Service) BeginInitParams(config []byte, dummy *int) error {
-	select {
-	case <-s.initialized:
-		return ErrAlreadyInitialized
-	default:
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.opt != nil {
-		s.opt.Cleanup()
-	}
-
-	// TODO(helin): parse learning rate from config
-	s.opt = newOptimizer(sgd, 0.01)
-	return nil
-}
-
 // InitParam initializes a parameter.
 func (s *Service) InitParam(paramWithConfigs ParameterWithConfig, dummy *int) error {
 	select {
@@ -109,75 +88,45 @@ func (s *Service) FinishInitParams(dummy0 int, dummy1 *int) error {
 	return nil
 }
 
-// SendGrads sends gradients to parameter servers for parameter
+// SendGrad sends gradient to parameter servers for parameter
 // optimization.
-func (s *Service) SendGrads(grads []Gradient, dummy *int) error {
+func (s *Service) SendGrad(g Gradient, dummy *int) error {
 	select {
 	case <-s.initialized:
 	default:
 		return ErrUninitialized
 	}
 
-	count := len(grads)
-	if count == 0 {
-		return nil
-	}
-
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	for _, g := range grads {
-		if _, ok := s.paramMap[g.Name]; !ok {
-			return fmt.Errorf("parameter: %s does not exist", g.Name)
-		}
-	}
-
-	errCh := make(chan error, count)
-	for _, g := range grads {
-		go func(p Parameter, g Gradient) {
-			err := s.opt.UpdateParameter(p, g)
-			errCh <- err
-		}(s.paramMap[g.Name], g)
+	p, ok := s.paramMap[g.Name]
+	if !ok {
+		return fmt.Errorf("parameter: %s does not exist", g.Name)
 	}
 
-	recv := 0
-	for err := range errCh {
-		if err != nil {
-			return err
-		}
-
-		recv++
-		if recv == count {
-			break
-		}
-	}
-	return nil
+	return s.opt.UpdateParameter(p, g)
 }
 
-// GetParams gets parameters from the parameter server.
-func (s *Service) GetParams(names []string, parameters *[]Parameter) error {
+// GetParam gets parameters from the parameter server.
+func (s *Service) GetParam(name string, parameter *Parameter) error {
 	<-s.initialized
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	for _, n := range names {
-		if _, ok := s.paramMap[n]; !ok {
-			return fmt.Errorf("parameter: %s does not exist", n)
-		}
-	}
-
-	*parameters = make([]Parameter, len(names))
-	for i, n := range names {
-		// The parameter content (a byte slice) may change
-		// during RPC serialization due to write from other
-		// goroutine, we allow it since mini-batch based deep
-		// learning optimization methods are stochastic in
-		// nature. This race condition is allowed deliberately
-		// to save the program from making a copy of the
-		// paramter content.
-		(*parameters)[i] = s.paramMap[n]
+	p, ok := s.paramMap[name]
+	if !ok {
+		return fmt.Errorf("parameter: %s does not exist", name)
 	}
 
+	// The parameter content (a byte slice) may change
+	// during RPC serialization due to write from other
+	// goroutine, we allow it since mini-batch based deep
+	// learning optimization methods are stochastic in
+	// nature. This race condition is allowed deliberately
+	// to save the program from making a copy of the
+	// paramter content.
+	*parameter = p
 	return nil
 }
 

paddle/go/pserver/service_test.go → go/pserver/service_test.go

@@ -4,23 +4,19 @@ import (
 	"reflect"
 	"sync"
 	"testing"
+	"time"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/pserver"
+	"github.com/PaddlePaddle/Paddle/go/pserver"
 )
 
 func TestFull(t *testing.T) {
 	s := pserver.NewService()
-	var dummy int
-	err := s.BeginInitParams(nil, &dummy)
-	if err != nil {
-		t.FailNow()
-	}
-
 	var p pserver.Parameter
 	p.Name = "param_a"
 	p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0}
 	p.ElementType = pserver.Int32
-	err = s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy)
+	var dummy int
+	err := s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy)
 	if err != nil {
 		t.FailNow()
 	}
@@ -39,40 +35,39 @@ func TestFull(t *testing.T) {
 		t.FailNow()
 	}
 
-	var params []pserver.Parameter
-	err = s.GetParams([]string{"param_b", "param_a"}, &params)
+	var param pserver.Parameter
+	err = s.GetParam("param_b", &param)
 	if err != nil {
 		t.FailNow()
 	}
 
-	if len(params) != 2 || !reflect.DeepEqual(params[0], p1) || !reflect.DeepEqual(params[0], p1) {
+	if !reflect.DeepEqual(param, p1) {
 		t.FailNow()
 	}
 
-	grads := []pserver.Gradient{pserver.Gradient(p1), pserver.Gradient(p)}
-	err = s.SendGrads(grads, &dummy)
+	g1, g2 := pserver.Gradient(p1), pserver.Gradient(p)
+	err = s.SendGrad(g1, &dummy)
 	if err != nil {
 		t.FailNow()
 	}
+	err = s.SendGrad(g2, &dummy)
 
-	var params1 []pserver.Parameter
-	err = s.GetParams([]string{"param_b", "param_a"}, &params1)
 	if err != nil {
 		t.FailNow()
 	}
 
-	if len(params) != 2 {
+	var param1 pserver.Parameter
+	err = s.GetParam("param_a", &param1)
+	if err != nil {
 		t.FailNow()
 	}
 
 	// don't compare content, since it's already changed by
 	// gradient update.
-	params1[0].Content = nil
-	params1[0].Content = nil
+	param1.Content = nil
 	p.Content = nil
-	p1.Content = nil
 
-	if !reflect.DeepEqual(params1[0], p1) || !reflect.DeepEqual(params1[0], p1) {
+	if !reflect.DeepEqual(param1, p) {
 		t.FailNow()
 	}
 }
@@ -80,19 +75,7 @@ func TestFull(t *testing.T) {
 func TestMultipleInit(t *testing.T) {
 	s := pserver.NewService()
 	var dummy int
-	err := s.BeginInitParams(nil, &dummy)
-	if err != nil {
-		t.FailNow()
-	}
-
-	// this is fine, it's possible for client to call init
-	// multiple times.
-	err = s.BeginInitParams(nil, &dummy)
-	if err != nil {
-		t.FailNow()
-	}
-
-	err = s.FinishInitParams(0, &dummy)
+	err := s.FinishInitParams(0, &dummy)
 	if err != nil {
 		t.FailNow()
 	}
@@ -101,17 +84,12 @@ func TestMultipleInit(t *testing.T) {
 	if err != pserver.ErrAlreadyInitialized {
 		t.FailNow()
 	}
-
-	err = s.BeginInitParams(nil, &dummy)
-	if err != pserver.ErrAlreadyInitialized {
-		t.FailNow()
-	}
 }
 
 func TestUninitialized(t *testing.T) {
 	s := pserver.NewService()
 	var dummy int
-	err := s.SendGrads(nil, &dummy)
+	err := s.SendGrad(pserver.Gradient{}, &dummy)
 	if err != pserver.ErrUninitialized {
 		t.FailNow()
 	}
@@ -123,8 +101,8 @@ func TestBlockUntilInitialized(t *testing.T) {
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
-		var params []pserver.Parameter
-		err := s.GetParams(nil, &params)
+		var param pserver.Parameter
+		err := s.GetParam("param_a", &param)
 		if err != nil {
 			t.FailNow()
 		}
@@ -143,11 +121,7 @@ func TestBlockUntilInitialized(t *testing.T) {
 		ch <- struct{}{}
 	}()
 
-	var dummy int
-	err := s.BeginInitParams(nil, &dummy)
-	if err != nil {
-		t.FailNow()
-	}
+	time.Sleep(50 * time.Millisecond)
 
 	select {
 	case <-ch:
@@ -156,6 +130,16 @@ func TestBlockUntilInitialized(t *testing.T) {
 	default:
 	}
 
+	var p pserver.Parameter
+	p.Name = "param_a"
+	p.Content = []byte{1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0}
+	p.ElementType = pserver.Int32
+	var dummy int
+	err := s.InitParam(pserver.ParameterWithConfig{p, nil}, &dummy)
+	if err != nil {
+		t.FailNow()
+	}
+
 	err = s.FinishInitParams(0, &dummy)
 	if err != nil {
 		t.FailNow()

paddle/go/recordio/README.md → go/recordio/README.md

@@ -8,6 +8,7 @@ w := recordio.NewWriter(f)
 w.Write([]byte("Hello"))
 w.Write([]byte("World!"))
 w.Close()
+f.Close()
 ```
 
 ## Read
@@ -18,6 +19,7 @@ w.Close()
    f, e := os.Open("a_file.recordio")
    idx, e := recordio.LoadIndex(f)
    fmt.Println("Total records: ", idx.Len())
+   f.Close()
    ```
 
 2. Create one or more scanner to read a range of records.  The
@@ -30,7 +32,8 @@ w.Close()
    for s.Scan() {
       fmt.Println(string(s.Record()))
    }
-   if s.Err() != nil && s.Err() != io.EOF {
+   if s.Err() != nil {
       log.Fatalf("Something wrong with scanning: %v", e)
    }
+   f.Close()
    ```

go/recordio/c/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+
+get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
+get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
+
+project(cxx_go C Go)
+
+include(golang)
+include(flags)
+
+go_library(recordio STATIC)
+add_subdirectory(test)

go/recordio/c/crecordio.go

+package main
+
+/*
+#include <string.h>
+
+typedef int reader;
+typedef int writer;
+*/
+import "C"
+
+import (
+	"log"
+	"os"
+	"strings"
+	"unsafe"
+
+	"github.com/PaddlePaddle/Paddle/go/recordio"
+)
+
+var nullPtr = unsafe.Pointer(uintptr(0))
+
+type writer struct {
+	w *recordio.Writer
+	f *os.File
+}
+
+type reader struct {
+	scanner *recordio.Scanner
+}
+
+func cArrayToSlice(p unsafe.Pointer, len int) []byte {
+	if p == nullPtr {
+		return nil
+	}
+
+	// create a Go clice backed by a C array, reference:
+	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
+	//
+	// Go garbage collector will not interact with this data, need
+	// to be freed properly.
+	return (*[1 << 30]byte)(p)[:len:len]
+}
+
+//export create_recordio_writer
+func create_recordio_writer(path *C.char) C.writer {
+	p := C.GoString(path)
+	f, err := os.Create(p)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+
+	w := recordio.NewWriter(f, -1, -1)
+	writer := &writer{f: f, w: w}
+	return addWriter(writer)
+}
+
+//export recordio_write
+func recordio_write(writer C.writer, buf *C.uchar, size C.int) C.int {
+	w := getWriter(writer)
+	b := cArrayToSlice(unsafe.Pointer(buf), int(size))
+	c, err := w.w.Write(b)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+
+	return C.int(c)
+}
+
+//export release_recordio_writer
+func release_recordio_writer(writer C.writer) {
+	w := removeWriter(writer)
+	w.w.Close()
+	w.f.Close()
+}
+
+//export create_recordio_reader
+func create_recordio_reader(path *C.char) C.reader {
+	p := C.GoString(path)
+	s, err := recordio.NewScanner(strings.Split(p, ",")...)
+	if err != nil {
+		log.Println(err)
+		return -1
+	}
+
+	r := &reader{scanner: s}
+	return addReader(r)
+}
+
+//export recordio_read
+func recordio_read(reader C.reader, record **C.uchar) C.int {
+	r := getReader(reader)
+	if r.scanner.Scan() {
+		buf := r.scanner.Record()
+		if len(buf) == 0 {
+			*record = (*C.uchar)(nullPtr)
+			return 0
+		}
+
+		size := C.int(len(buf))
+		*record = (*C.uchar)(C.malloc(C.size_t(len(buf))))
+		C.memcpy(unsafe.Pointer(*record), unsafe.Pointer(&buf[0]), C.size_t(len(buf)))
+		return size
+	}
+
+	return -1
+}
+
+//export release_recordio_reader
+func release_recordio_reader(reader C.reader) {
+	r := removeReader(reader)
+	r.scanner.Close()
+}
+
+func main() {} // Required but ignored

go/recordio/c/register.go

+package main
+
+/*
+typedef int reader;
+typedef int writer;
+*/
+import "C"
+
+import "sync"
+
+var mu sync.Mutex
+var handleMap = make(map[C.reader]*reader)
+var curHandle C.reader
+var writerMap = make(map[C.writer]*writer)
+var curWriterHandle C.writer
+
+func addReader(r *reader) C.reader {
+	mu.Lock()
+	defer mu.Unlock()
+	reader := curHandle
+	curHandle++
+	handleMap[reader] = r
+	return reader
+}
+
+func getReader(reader C.reader) *reader {
+	mu.Lock()
+	defer mu.Unlock()
+	return handleMap[reader]
+}
+
+func removeReader(reader C.reader) *reader {
+	mu.Lock()
+	defer mu.Unlock()
+	r := handleMap[reader]
+	delete(handleMap, reader)
+	return r
+}
+
+func addWriter(w *writer) C.writer {
+	mu.Lock()
+	defer mu.Unlock()
+	writer := curWriterHandle
+	curWriterHandle++
+	writerMap[writer] = w
+	return writer
+}
+
+func getWriter(writer C.writer) *writer {
+	mu.Lock()
+	defer mu.Unlock()
+	return writerMap[writer]
+}
+
+func removeWriter(writer C.writer) *writer {
+	mu.Lock()
+	defer mu.Unlock()
+	w := writerMap[writer]
+	delete(writerMap, writer)
+	return w
+}

go/recordio/c/test/CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+
+include_directories(${CMAKE_BINARY_DIR})
+
+add_executable(recordio_test test.c)
+add_dependencies(recordio_test recordio)
+set (CMAKE_EXE_LINKER_FLAGS "-pthread")
+target_link_libraries(recordio_test ${CMAKE_BINARY_DIR}/librecordio.a)

go/recordio/c/test/test.c

+#include <stdio.h>
+#include <stdlib.h>
+
+#include "librecordio.h"
+
+void fail() {
+  // TODO(helin): fix: gtest using cmake is not working, using this
+  // hacky way for now.
+  printf("test failed.\n");
+  exit(-1);
+}
+
+int main() {
+  writer w = create_recordio_writer("/tmp/test_recordio_0");
+  recordio_write(w, "hello", 6);
+  recordio_write(w, "hi", 3);
+  release_recordio_writer(w);
+
+  w = create_recordio_writer("/tmp/test_recordio_1");
+  recordio_write(w, "dog", 4);
+  recordio_write(w, "cat", 4);
+  release_recordio_writer(w);
+
+  reader r = create_recordio_reader("/tmp/test_recordio_*");
+  unsigned char* item = NULL;
+  int size = recordio_read(r, &item);
+  if (strcmp(item, "hello") || size != 6) {
+    fail();
+  }
+  free(item);
+
+  size = recordio_read(r, &item);
+  if (strcmp(item, "hi") || size != 3) {
+    fail();
+  }
+  free(item);
+
+  size = recordio_read(r, &item);
+  if (strcmp(item, "dog") || size != 4) {
+    fail();
+  }
+  free(item);
+
+  size = recordio_read(r, &item);
+  if (strcmp(item, "cat") || size != 4) {
+    fail();
+  }
+  free(item);
+
+  size = recordio_read(r, &item);
+  if (size != -1) {
+    fail();
+  }
+
+  release_recordio_reader(r);
+}

paddle/go/recordio/reader.go → go/recordio/range_scanner.go

@@ -74,8 +74,8 @@ func (r *Index) Locate(recordIndex int) (int, int) {
 	return -1, -1
 }
 
-// Scanner scans records in a specified range within [0, numRecords).
-type Scanner struct {
+// RangeScanner scans records in a specified range within [0, numRecords).
+type RangeScanner struct {
 	reader          io.ReadSeeker
 	index           *Index
 	start, end, cur int
@@ -84,10 +84,10 @@ type Scanner struct {
 	err             error
 }
 
-// NewScanner creates a scanner that sequencially reads records in the
+// NewRangeScanner creates a scanner that sequencially reads records in the
 // range [start, start+len).  If start < 0, it scans from the
 // beginning.  If len < 0, it scans till the end of file.
-func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner {
+func NewRangeScanner(r io.ReadSeeker, index *Index, start, len int) *RangeScanner {
 	if start < 0 {
 		start = 0
 	}
@@ -95,7 +95,7 @@ func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner {
 		len = index.NumRecords() - start
 	}
 
-	return &Scanner{
+	return &RangeScanner{
 		reader:     r,
 		index:      index,
 		start:      start,
@@ -108,7 +108,7 @@ func NewScanner(r io.ReadSeeker, index *Index, start, len int) *Scanner {
 
 // Scan moves the cursor forward for one record and loads the chunk
 // containing the record if not yet.
-func (s *Scanner) Scan() bool {
+func (s *RangeScanner) Scan() bool {
 	s.cur++
 
 	if s.cur >= s.end {
@@ -124,12 +124,17 @@ func (s *Scanner) Scan() bool {
 }
 
 // Record returns the record under the current cursor.
-func (s *Scanner) Record() []byte {
+func (s *RangeScanner) Record() []byte {
 	_, ri := s.index.Locate(s.cur)
 	return s.chunk.records[ri]
 }
 
-// Error returns the error that stopped Scan.
-func (s *Scanner) Error() error {
+// Err returns the first non-EOF error that was encountered by the
+// Scanner.
+func (s *RangeScanner) Err() error {
+	if s.err == io.EOF {
+		return nil
+	}
+
 	return s.err
 }

paddle/go/recordio/recordio_internal_test.go → go/recordio/recordio_internal_test.go

@@ -68,7 +68,7 @@ func TestWriteAndRead(t *testing.T) {
 				2*4)}, // two record legnths
 		idx.chunkOffsets)
 
-	s := NewScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1)
+	s := NewRangeScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1)
 	i := 0
 	for s.Scan() {
 		assert.Equal(data[i], string(s.Record()))

paddle/go/recordio/recordio_test.go → go/recordio/recordio_test.go

@@ -5,7 +5,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/PaddlePaddle/Paddle/paddle/go/recordio"
+	"github.com/PaddlePaddle/Paddle/go/recordio"
 )
 
 func TestWriteRead(t *testing.T) {
@@ -29,7 +29,7 @@ func TestWriteRead(t *testing.T) {
 		t.Fatal("num record does not match:", idx.NumRecords(), total)
 	}
 
-	s := recordio.NewScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1)
+	s := recordio.NewRangeScanner(bytes.NewReader(buf.Bytes()), idx, -1, -1)
 	i := 0
 	for s.Scan() {
 		if !reflect.DeepEqual(s.Record(), make([]byte, i)) {
@@ -66,7 +66,7 @@ func TestChunkIndex(t *testing.T) {
 
 	for i := 0; i < total; i++ {
 		newIdx := idx.ChunkIndex(i)
-		s := recordio.NewScanner(bytes.NewReader(buf.Bytes()), newIdx, -1, -1)
+		s := recordio.NewRangeScanner(bytes.NewReader(buf.Bytes()), newIdx, -1, -1)
 		j := 0
 		for s.Scan() {
 			if !reflect.DeepEqual(s.Record(), make([]byte, i)) {

go/recordio/scanner.go

+package recordio
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Scanner is a scanner for multiple recordio files.
+type Scanner struct {
+	paths      []string
+	curFile    *os.File
+	curScanner *RangeScanner
+	pathIdx    int
+	end        bool
+	err        error
+}
+
+// NewScanner creates a new Scanner.
+func NewScanner(paths ...string) (*Scanner, error) {
+	var ps []string
+	for _, s := range paths {
+		match, err := filepath.Glob(s)
+		if err != nil {
+			return nil, err
+		}
+
+		ps = append(ps, match...)
+	}
+
+	if len(ps) == 0 {
+		return nil, fmt.Errorf("no valid path provided: %v", paths)
+	}
+
+	return &Scanner{paths: ps}, nil
+}
+
+// Scan moves the cursor forward for one record and loads the chunk
+// containing the record if not yet.
+func (s *Scanner) Scan() bool {
+	if s.err != nil {
+		return false
+	}
+
+	if s.end {
+		return false
+	}
+
+	if s.curScanner == nil {
+		more, err := s.nextFile()
+		if err != nil {
+			s.err = err
+			return false
+		}
+
+		if !more {
+			s.end = true
+			return false
+		}
+	}
+
+	curMore := s.curScanner.Scan()
+	s.err = s.curScanner.Err()
+
+	if s.err != nil {
+		return curMore
+	}
+
+	if !curMore {
+		err := s.curFile.Close()
+		if err != nil {
+			s.err = err
+			return false
+		}
+		s.curFile = nil
+
+		more, err := s.nextFile()
+		if err != nil {
+			s.err = err
+			return false
+		}
+
+		if !more {
+			s.end = true
+			return false
+		}
+
+		return s.Scan()
+	}
+	return true
+}
+
+// Err returns the first non-EOF error that was encountered by the
+// Scanner.
+func (s *Scanner) Err() error {
+	return s.err
+}
+
+// Record returns the record under the current cursor.
+func (s *Scanner) Record() []byte {
+	if s.curScanner == nil {
+		return nil
+	}
+
+	return s.curScanner.Record()
+}
+
+// Close release the resources.
+func (s *Scanner) Close() error {
+	s.curScanner = nil
+	if s.curFile != nil {
+		err := s.curFile.Close()
+		s.curFile = nil
+		return err
+	}
+	return nil
+}
+
+func (s *Scanner) nextFile() (bool, error) {
+	if s.pathIdx >= len(s.paths) {
+		return false, nil
+	}
+
+	path := s.paths[s.pathIdx]
+	s.pathIdx++
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+
+	idx, err := LoadIndex(f)
+	if err != nil {
+		f.Close()
+		return false, err
+	}
+
+	s.curFile = f
+	s.curScanner = NewRangeScanner(f, idx, 0, -1)
+	return true, nil
+}

paddle/CMakeLists.txt

@@ -9,9 +9,10 @@ add_subdirectory(pserver)
 add_subdirectory(trainer)
 add_subdirectory(scripts)
 
-if(CMAKE_Go_COMPILER)
-  add_subdirectory(go)
-endif()
+# Do not build go directory until go cmake is working smoothly.
+# if(CMAKE_Go_COMPILER)
+#   add_subdirectory(go)
+# endif()
 
 find_package(Boost QUIET)
 

paddle/go/CMakeLists.txt

-include_directories(${CMAKE_CURRENT_BINARY_DIR})
-
-go_library(adder SRCS adder.go)
-
-if (WITH_TESTING)
-  cc_test(cgo_test
-        SRCS
-        cgo_test.cc
-        DEPS
-        adder)
-endif()

paddle/go/adder.go

-package main
-
-import "C"
-
-//export GoAdder
-func GoAdder(x, y int) int {
-	return x + y
-}
-
-func main() {} // Required but ignored

paddle/go/cclient/CMakeLists.txt

-cmake_minimum_required(VERSION 3.0)
-
-if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
-  message("-- Found gtest (include: ${GTEST_INCLUDE_DIR}, library: ${GTEST_LIBRARIES})")
-else()
-  # find cmake directory modules
-  get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
-  get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
-  get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
-
-  set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
-
-  # enable c++11
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-
-  # enable gtest
-  set(THIRD_PARTY_PATH ./third_party)
-  set(WITH_TESTING ON)
-  include(external/gtest)
-endif()
-
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-
-project(cxx_go CXX C Go)
-
-include(cmake/golang.cmake)
-include(cmake/flags.cmake)
-
-ExternalGoProject_Add(pserver github.com/PaddlePaddle/Paddle/paddle/go/pserver)
-add_go_library(client STATIC pserver)
-add_subdirectory(test)

paddle/go/cgo_test.cc

-#include <iostream>
-#include "gtest/gtest.h"
-#include "libadder.h"
-
-TEST(Cgo, Invoke) { EXPECT_EQ(GoAdder(30, 12), 42); }

paddle/go/pserver/client.go

-package pserver
-
-// Client is the client to parameter servers.
-type Client struct {
-}
-
-// NewClient creates a new client.
-func NewClient(addr string) *Client {
-	return &Client{}
-}
-
-// BeginInitParams begins to initialize parameters on parameter
-// servers.
-//
-// BeginInitParams will be called from multiple trainers, only one
-// trainer will be selected to initialize the parameters on parameter
-// servers. Other trainers will be blocked until the initialization is
-// done, and they need to get the initialized parameters from
-// parameter servers using GetParams.
-func (c *Client) BeginInitParams(pserverConfigProto []byte) (selected bool, err error) {
-	return true, nil
-}
-
-// InitParam initializes the parameter on parameter servers.
-func (c *Client) InitParam(paramWithConfigs ParameterWithConfig) error {
-	return nil
-}
-
-// FinishInitParams tells parameter servers client has sent all
-// parameters to parameter servers as initialization.
-func (c *Client) FinishInitParams() error {
-	return nil
-}
-
-// SendGrads sends gradients to parameter servers for updating
-// parameters.
-func (c *Client) SendGrads(grads []Gradient) error {
-	return nil
-}
-
-// GetParams gets parameters from parameter servers.
-func (c *Client) GetParams(names []string) ([]Parameter, error) {
-	return nil, nil
-}
-
-// SaveModel indicates parameters to save the parameter to the given
-// path.
-func (c *Client) SaveModel(path string) error {
-	return nil
-}
-
-// Cleanup cleans up the client states.
-func (c *Client) Cleanup() {
-}

paddle/gserver/layers/CostLayer.cpp

@@ -217,10 +217,10 @@ void SmoothL1CostLayer::forwardImp(Matrix& output,
     targetCpu->copyFrom(target);
     outputCpu->copyFrom(output);
     labelCpu->copyFrom(*label.value);
-    targetCpu->smoothL1(*outputCpu, *labelCpu);
+    targetCpu->smoothL1(*outputCpu, *labelCpu, 1.0);
     target.copyFrom(*targetCpu);
   } else {
-    target.smoothL1(output, *label.value);
+    target.smoothL1(output, *label.value, 1.0);
   }
 }
 
@@ -238,10 +238,10 @@ void SmoothL1CostLayer::backwardImp(Matrix& output,
     outputGCpu->copyFrom(outputG);
     outputCpu->copyFrom(output);
     labelCpu->copyFrom(*label.value);
-    outputGCpu->smoothL1Bp(*outputCpu, *labelCpu);
+    outputGCpu->smoothL1Bp(*outputCpu, *labelCpu, 1.0);
     outputG.copyFrom(*outputGCpu);
   } else {
-    outputG.smoothL1Bp(output, *label.value);
+    outputG.smoothL1Bp(output, *label.value, 1.0);
   }
 }
 

paddle/gserver/tests/sequence_nest_layer_group.conf

@@ -59,7 +59,7 @@ lstm_nest_group = recurrent_group(
     input=SubsequenceInput(emb_group), step=lstm_group, name="lstm_nest_group")
 # hasSubseq ->(seqlastins) seq
 lstm_last = last_seq(
-    input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE)
+    input=lstm_nest_group, agg_level=AggregateLevel.TO_SEQUENCE)
 
 # seq ->(expand) hasSubseq
 lstm_expand = expand_layer(
@@ -71,7 +71,7 @@ lstm_expand = expand_layer(
 lstm_average = pooling_layer(
     input=lstm_expand,
     pooling_type=AvgPooling(),
-    agg_level=AggregateLevel.EACH_SEQUENCE)
+    agg_level=AggregateLevel.TO_SEQUENCE)
 
 with mixed_layer(
         size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output:

paddle/majel/CMakeLists.txt

 cc_library(place SRCS place.cc)
+cc_test(place_test SRCS place_test.cc DEPS place glog gflags)
+
 cc_library(ddim SRCS ddim.cc)
+cc_test(ddim_test SRCS ddim_test.cc DEPS ddim)
 
-if(WITH_TESTING)
-    add_subdirectory(test)
-endif()
+nv_test(cuda_test SRCS cuda_test.cu)
+nv_test(dim_test SRCS dim_test.cu DEPS ddim)

paddle/majel/test/place_test.cc → paddle/majel/place_test.cc

 #include "paddle/majel/place.h"
 #include <sstream>
 #include "gtest/gtest.h"
-#include "paddle/utils/Logging.h"
 
 TEST(Place, Equality) {
   majel::CpuPlace cpu;
@@ -38,5 +37,4 @@ TEST(Place, Print) {
     ss << majel::CpuPlace();
     EXPECT_EQ("CpuPlace", ss.str());
   }
-  LOG(INFO) << "\n[----------] Done \n";
 }

paddle/majel/test/CMakeLists.txt

-cc_test(place_test
-    SRCS place_test.cc
-    DEPS place)
-
-cc_test(ddim_test
-    SRCS ddim_test.cc
-    DEPS ddim)
-
-if(WITH_GPU)
-    nv_test(cuda_test SRCS cuda_test.cu)
-    nv_test(dim_test SRCS dim_test.cu DEPS ddim)
-endif()

paddle/math/Matrix.cpp

@@ -3606,7 +3606,7 @@ void CpuMatrix::sumOfSquaresBp(Matrix& output, Matrix& label) {
   }
 }
 
-void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
+void CpuMatrix::smoothL1(Matrix& output, Matrix& label, real destScale) {
   CHECK(output.useGpu_ == false && label.useGpu_ == false)
       << "Matrix type are not equal";
 
@@ -3624,6 +3624,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
   for (size_t i = 0; i < numSamples; ++i, out += dim, lbl += dim) {
     for (size_t j = 0; j < dim; ++j) {
       real absVal = std::fabs(out[j] - lbl[j]);
+      cost[i] *= destScale;
       if (absVal < 1.0)
         cost[i] += 0.5 * absVal * absVal;
       else
@@ -3632,7 +3633,7 @@ void CpuMatrix::smoothL1(Matrix& output, Matrix& label) {
   }
 }
 
-void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) {
+void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label, real destScale) {
   CHECK(output.useGpu_ == false && label.useGpu_ == false)
       << "Matrix type are not equal";
 
@@ -3650,6 +3651,7 @@ void CpuMatrix::smoothL1Bp(Matrix& output, Matrix& label) {
   for (size_t i = 0; i < numSamples; ++i, out += dim, grad += dim, lbl += dim) {
     for (size_t j = 0; j < dim; ++j) {
       real val = out[j] - lbl[j];
+      grad[j] *= destScale;
       if (std::fabs(val) < 1) {
         grad[j] += val;
       } else {

paddle/math/Matrix.h

@@ -789,11 +789,11 @@ public:
     LOG(FATAL) << "Not implemented";
   }
 
-  virtual void smoothL1(Matrix& output, Matrix& label) {
+  virtual void smoothL1(Matrix& output, Matrix& label, real destScale) {
     LOG(FATAL) << "Not implemented";
   }
 
-  virtual void smoothL1Bp(Matrix& outputV, Matrix& label) {
+  virtual void smoothL1Bp(Matrix& outputV, Matrix& label, real destScale) {
     LOG(FATAL) << "Not implemented";
   }
 
@@ -1736,8 +1736,8 @@ public:
   /// gradient of sumOfSquares.
   void sumOfSquaresBp(Matrix& outputV, Matrix& label);
 
-  void smoothL1(Matrix& output, Matrix& label);
-  void smoothL1Bp(Matrix& output, Matrix& label);
+  void smoothL1(Matrix& output, Matrix& label, real destScale);
+  void smoothL1Bp(Matrix& output, Matrix& label, real destScale);
 
   void tanh(Matrix& output);
   void tanhDerivative(Matrix& output);

paddle/parameter/Parameter.h

@@ -324,6 +324,7 @@ protected:
   std::vector<std::shared_ptr<IParameterUpdaterHook>> updaterHooks_;
 
 public:
+  void setSharedCount(int cnt) { sharedCount_ = cnt; }
   int getSharedCount() { return sharedCount_; }
 
   bool isSparse() { return config_.is_sparse(); }

paddle/scripts/docker/build.sh

@@ -21,6 +21,8 @@ cd /paddle/build
 
 # build script will not fail if *.deb does not exist
 rm *.deb 2>/dev/null || true
+# delete previous built whl packages
+rm -rf /paddle/paddle/dist 2>/dev/null || true
 
 cat <<EOF
 ========================================

paddle/utils/CpuId.cpp

@@ -21,7 +21,7 @@ limitations under the License. */
 
 #else
 
-#if !defined(__arm__)
+#if !defined(__arm__) && !defined(__aarch64__)
 #include <cpuid.h>
 /// for GCC/Clang
 #define CPUID(info, x) __cpuid_count(x, 0, info[0], info[1], info[2], info[3])
@@ -32,7 +32,7 @@ limitations under the License. */
 namespace paddle {
 
 SIMDFlags::SIMDFlags() {
-#if defined(__arm__)
+#if defined(__arm__) || defined(__aarch64__)
   simd_flags_ = SIMD_NEON;
 #else
   unsigned int cpuInfo[4];

paddle/utils/arch/linux/Locks.cpp

@@ -55,8 +55,11 @@ public:
 };
 
 #else
-
+// clang-format off
+#include <cstddef>
 #include <atomic>
+// clang-format on
+
 class SpinLockPrivate {
 public:
   inline void lock() {

paddle/utils/tests/test_SIMDFlags.cpp

@@ -19,7 +19,7 @@ using namespace paddle;  // NOLINT
 
 TEST(SIMDFlags, gccTest) {
 #if (defined(__GNUC__) || defined(__GNUG__)) && !(defined(__clang__)) && \
-    !defined(__arm__)
+    !defined(__arm__) && !defined(__aarch64__)
   // clang-format off
   CHECK(!__builtin_cpu_supports("sse")    != HAS_SSE);
   CHECK(!__builtin_cpu_supports("sse2")   != HAS_SSE2);

python/paddle/trainer/config_parser.py

@@ -3378,7 +3378,7 @@ def make_importer(config_dir, config_args):
     return Import
 
 
-settings = dict(
+DEFAULT_SETTING = dict(
     batch_size=None,
     mini_batch_size=None,
     algorithm='async_sgd',
@@ -3411,6 +3411,8 @@ settings = dict(
     adam_beta2=0.999,
     adam_epsilon=1e-8, )
 
+settings = copy.deepcopy(DEFAULT_SETTING)
+
 settings_deprecated = dict(usage_ratio=1., )
 
 trainer_settings = dict(
@@ -3551,10 +3553,8 @@ def update_g_config():
     return g_config
 
 
-def parse_config(trainer_config, config_arg_str):
+def begin_parse(config_arg_str=''):
     '''
-    @param trainer_config: can be a string of config file name or a function name
-    with config logic
     @param config_arg_str: a string of the form var1=val1,var2=val2. It will be
     passed to config script as a dictionary CONFIG_ARGS
     '''
@@ -3562,12 +3562,23 @@ def parse_config(trainer_config, config_arg_str):
     for hook in _parse_config_hooks:
         hook()
 
-    config_args = {}
-
     logger.findCaller = find_caller
     logger.fatal = my_fatal
 
     g_config.model_config.type = "nn"
+
+    global g_current_submodel, g_root_submodel
+    g_root_submodel = g_config.model_config.sub_models.add()
+    g_root_submodel.name = 'root'
+    g_root_submodel.is_recurrent_layer_group = False
+    g_current_submodel = g_root_submodel
+
+
+def parse_config(trainer_config, config_arg_str):
+    begin_parse(config_arg_str)
+
+    config_args = {}
+
     if config_arg_str:
         config_args = dict([f.split('=') for f in config_arg_str.split(',')])
 
@@ -3580,14 +3591,6 @@ def parse_config(trainer_config, config_arg_str):
         extension_module = importlib(extension_module_name)
         g_extended_config_funcs = extension_module.get_config_funcs(g_config)
 
-    g_config.model_config.type = 'nn'
-
-    global g_current_submodel, g_root_submodel
-    g_root_submodel = g_config.model_config.sub_models.add()
-    g_root_submodel.name = 'root'
-    g_root_submodel.is_recurrent_layer_group = False
-    g_current_submodel = g_root_submodel
-
     if hasattr(trainer_config, '__call__'):
         trainer_config.func_globals.update(
             make_config_environment("", config_args))

python/paddle/trainer_config_helpers/config_parser_utils.py

@@ -12,15 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import paddle.trainer.config_parser as config_parser
+from paddle.proto.TrainerConfig_pb2 import OptimizationConfig
 '''
-This file is a wrapper of formal config_parser. The main idea of this file is to 
+This file is a wrapper of formal config_parser. The main idea of this file is to
 separete different config logic into different function, such as network configuration
  and optimizer configuration.
 '''
 
 __all__ = [
-    "parse_trainer_config", "parse_network_config", "parse_optimizer_config"
+    "parse_trainer_config", "parse_network_config", "parse_optimizer_config",
+    "reset_parser"
 ]
 
 
@@ -34,5 +37,15 @@ def parse_network_config(network_conf, config_arg_str=''):
 
 
 def parse_optimizer_config(optimizer_conf, config_arg_str=''):
-    config = config_parser.parse_config(optimizer_conf, config_arg_str)
-    return config.opt_config
+    config_parser.settings = copy.deepcopy(config_parser.DEFAULT_SETTING)
+    optimizer_conf()
+    opt_config = OptimizationConfig()
+    for k, v in config_parser.settings.iteritems():
+        if v is None:
+            continue
+        opt_config.__setattr__(k, v)
+    return opt_config
+
+
+def reset_parser():
+    config_parser.begin_parse()

python/paddle/trainer_config_helpers/layers.py

@@ -119,6 +119,7 @@ __all__ = [
     'eos_layer',
     'smooth_l1_cost',
     'layer_support',
+    'multiplex_layer',
 ]
 
 
@@ -185,6 +186,7 @@ class LayerType(object):
     MAXOUT = "maxout"
     SPP_LAYER = "spp"
     PAD_LAYER = "pad"
+    MULTIPLEX_LAYER = "multiplex"
 
     PRINT_LAYER = "print"
     PRIORBOX_LAYER = "priorbox"
@@ -235,16 +237,19 @@ class AggregateLevel(object):
 
     Accordingly, AggregateLevel supports two modes:
 
-    - :code:`AggregateLevel.EACH_TIMESTEP` means the aggregation acts on each
+    - :code:`AggregateLevel.TO_NO_SEQUENCE` means the aggregation acts on each
       timestep of a sequence, both :code:`SUB_SEQUENCE` and :code:`SEQUENCE` will
       be aggregated to :code:`NO_SEQUENCE`.
 
-    - :code:`AggregateLevel.EACH_SEQUENCE` means the aggregation acts on each
+    - :code:`AggregateLevel.TO_SEQUENCE` means the aggregation acts on each
       sequence of a nested sequence, :code:`SUB_SEQUENCE` will be aggregated to
       :code:`SEQUENCE`.
     """
-    EACH_TIMESTEP = 'non-seq'
-    EACH_SEQUENCE = 'seq'
+    TO_NO_SEQUENCE = 'non-seq'
+    TO_SEQUENCE = 'seq'
+    # compatible with previous configuration
+    EACH_TIMESTEP = TO_NO_SEQUENCE
+    EACH_SEQUENCE = TO_SEQUENCE
 
 
 class LayerOutput(object):
@@ -285,6 +290,7 @@ class LayerOutput(object):
         assert size is not None
         assert LayerType.is_layer_type(layer_type)
         self.name = name
+        self.full_name = MakeLayerNameInSubmodel(name)
         self.layer_type = layer_type
         if parents is not None and type(parents) != list:
             parents = [parents]
@@ -479,7 +485,7 @@ def table_projection(input, size=0, param_attr=None):
     return proj
 
 
-def identity_projection(input, offset=None):
+def identity_projection(input, offset=None, size=None):
     """
     1. IdentityProjection if offset=None. It performs:
 
@@ -520,8 +526,10 @@ def identity_projection(input, offset=None):
         proj = IdentityProjection(input_layer_name=input.name)
         proj.origin = input
     else:
+        if size is None:
+            size = input.size - offset
         proj = IdentityOffsetProjection(
-            input_layer_name=input.name, offset=offset)
+            input_layer_name=input.name, offset=offset, size=size)
         proj.origin = input
     return proj
 
@@ -1078,7 +1086,7 @@ def pooling_layer(input,
                   pooling_type=None,
                   name=None,
                   bias_attr=None,
-                  agg_level=AggregateLevel.EACH_TIMESTEP,
+                  agg_level=AggregateLevel.TO_NO_SEQUENCE,
                   layer_attr=None):
     """
     Pooling layer for sequence inputs, not used for Image.
@@ -1089,10 +1097,10 @@ def pooling_layer(input,
 
        seq_pool = pooling_layer(input=layer,
                                 pooling_type=AvgPooling(),
-                                agg_level=AggregateLevel.EACH_SEQUENCE)
+                                agg_level=AggregateLevel.TO_NO_SEQUENCE)
 
-    :param agg_level: AggregateLevel.EACH_TIMESTEP or
-                      AggregateLevel.EACH_SEQUENCE
+    :param agg_level: AggregateLevel.TO_NO_SEQUENCE or
+                      AggregateLevel.TO_SEQUENCE
     :type agg_level: AggregateLevel
     :param name: layer name.
     :type name: basestring
@@ -1362,7 +1370,7 @@ def grumemory(input,
 @layer_support()
 def last_seq(input,
              name=None,
-             agg_level=AggregateLevel.EACH_TIMESTEP,
+             agg_level=AggregateLevel.TO_NO_SEQUENCE,
              stride=-1,
              layer_attr=None):
     """
@@ -1397,7 +1405,7 @@ def last_seq(input,
                        " series information at all. Maybe you want to use"
                        " first_seq instead.")
 
-    if agg_level == AggregateLevel.EACH_SEQUENCE:
+    if agg_level == AggregateLevel.TO_SEQUENCE:
         assert stride == -1
 
     Layer(
@@ -1418,7 +1426,7 @@ def last_seq(input,
 @layer_support()
 def first_seq(input,
               name=None,
-              agg_level=AggregateLevel.EACH_TIMESTEP,
+              agg_level=AggregateLevel.TO_NO_SEQUENCE,
               stride=-1,
               layer_attr=None):
     """
@@ -1454,7 +1462,7 @@ def first_seq(input,
                        ' time series information at all. Maybe you want to use'
                        ' last_seq instead.')
 
-    if agg_level == AggregateLevel.EACH_SEQUENCE:
+    if agg_level == AggregateLevel.TO_SEQUENCE:
         assert stride == -1
 
     Layer(
@@ -1477,16 +1485,18 @@ class ExpandLevel(object):
 
     ExpandLevel supports two modes:
 
-    - :code:`ExpandLevel.FROM_TIMESTEP` means the expandation acts on each
-      timestep of a sequence, :code:`NO_SEQUENCE` will be expanded to
+    - :code:`ExpandLevel.FROM_NO_SEQUENCE` means the expansion acts on
+      :code:`NO_SEQUENCE`, which will be expanded to
       :code:`SEQUENCE` or :code:`SUB_SEQUENCE`.
 
-    - :code:`ExpandLevel.FROM_SEQUENCE` means the expandation acts on each
-      sequence of a nested sequence, :code:`SEQUENCE` will be expanded to
+    - :code:`ExpandLevel.FROM_SEQUENCE` means the expansion acts on
+      :code:`SEQUENCE`, which will be expanded to
       :code:`SUB_SEQUENCE`.
     """
-    FROM_TIMESTEP = AggregateLevel.EACH_TIMESTEP
-    FROM_SEQUENCE = AggregateLevel.EACH_SEQUENCE
+    FROM_NO_SEQUENCE = AggregateLevel.TO_NO_SEQUENCE
+    FROM_SEQUENCE = AggregateLevel.TO_SEQUENCE
+    # compatible with previous configuration
+    FROM_TIMESTEP = FROM_NO_SEQUENCE
 
 
 @wrap_name_default()
@@ -1495,7 +1505,7 @@ def expand_layer(input,
                  expand_as,
                  name=None,
                  bias_attr=False,
-                 expand_level=ExpandLevel.FROM_TIMESTEP,
+                 expand_level=ExpandLevel.FROM_NO_SEQUENCE,
                  layer_attr=None):
     """
     A layer for "Expand Dense data or (sequence data where the length of each
@@ -1507,7 +1517,7 @@ def expand_layer(input,
 
        expand = expand_layer(input=layer1,
                              expand_as=layer2,
-                             expand_level=ExpandLevel.FROM_TIMESTEP)
+                             expand_level=ExpandLevel.FROM_NO_SEQUENCE)
 
     :param input: Input layer
     :type input: LayerOutput
@@ -2794,7 +2804,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None):
     if layer_type == LayerType.CONCAT_LAYER:
         assert not bias_attr
 
-    Layer(
+    layer = Layer(
         name=name,
         type=layer_type,
         inputs=[x.name for x in input] if is_concat_layer else input,
@@ -2802,13 +2812,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None):
         bias=ParamAttr.to_bias(bias_attr),
         **ExtraLayerAttribute.to_kwargs(layer_attr))
 
-    sz = 0
-    for each_input in input:
-        if each_input.size is not None:
-            sz += each_input.size
-        else:
-            sz = None
-            break
+    sz = layer.config.size
 
     return LayerOutput(
         name,
@@ -2976,7 +2980,7 @@ def memory(name,
 @layer_support()
 def lstm_step_layer(input,
                     state,
-                    size,
+                    size=None,
                     act=None,
                     name=None,
                     gate_act=None,
@@ -3042,6 +3046,9 @@ def lstm_step_layer(input,
     :return: LayerOutput object.
     :rtype: LayerOutput
     """
+
+    assert size is None or state.size == size
+    size = state.size
     Layer(
         name=name,
         type=LayerType.LSTM_STEP_LAYER,
@@ -3049,7 +3056,7 @@ def lstm_step_layer(input,
         active_gate_type=gate_act.name,
         active_state_type=state_act.name,
         bias=ParamAttr.to_bias(bias_attr),
-        size=size,
+        size=state.size,
         inputs=[input.name, state.name],
         **ExtraLayerAttribute.to_kwargs(layer_attr))
 
@@ -3489,6 +3496,11 @@ def recurrent_group(step,
 
     RecurrentLayerGroupEnd(name=name)
 
+    for layer_out in layer_outs:
+        # Thee previous full_name is the name is the rnn group
+        # We need a full_name outside the rnn group
+        layer_out.full_name = MakeLayerNameInSubmodel(layer_out.name)
+
     if len(layer_outs) == 1:
         return layer_outs[0]
     else:
@@ -5465,3 +5477,54 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None):
         **ExtraLayerAttribute.to_kwargs(layer_attr))
     return LayerOutput(
         name, LayerType.SMOOTH_L1, parents=[input, label], size=1)
+
+
+@wrap_name_default()
+def multiplex_layer(input, name=None, layer_attr=None):
+    """
+    This layer multiplex multiple layers according to the index,
+    which is provided by the first input layer.
+    inputs[0]: the index of the layer to output of size batchSize.
+    inputs[1:N]; the candidate output data.
+    For each index i from 0 to batchSize -1, the output is the i-th row of the
+    (index[i] + 1)-th layer.
+
+    For each i-th row of output:
+    .. math::
+        y[i][j] = x_{x_{0}[i] + 1}[i][j], j = 0,1, ... , (x_{1}.width - 1)
+
+    where, y is output. :math:`x_{k}` is the k-th input layer and
+    :math:`k = x_{0}[i] + 1`.
+
+    .. code-block:: python
+
+       maxid = multiplex_layer(input=layers)
+
+    :param input: Input layers.
+    :type input: list of LayerOutput
+    :param name: Layer name.
+    :type name: basestring
+    :param layer_attr: extra layer attributes.
+    :type layer_attr: ExtraLayerAttribute.
+    :return: LayerOutput object.
+    :rtype: LayerOutput
+    """
+
+    assert isinstance(input, collections.Sequence)
+    assert len(input) > 2, 'multiplex_layer should have more than 2 inputs'
+    for i in range(1, len(input)):
+        assert isinstance(input[i], LayerOutput)
+        assert input[i].size == input[1].size, \
+            "All the input layers except the first one should have the same size"
+
+    l = Layer(
+        name=name,
+        type='multiplex',
+        inputs=[x.name for x in input],
+        size=input[1].size,
+        **ExtraLayerAttribute.to_kwargs(layer_attr))
+    return LayerOutput(
+        name=name,
+        layer_type=LayerType.MULTIPLEX_LAYER,
+        parents=input,
+        size=l.config.size)

python/paddle/trainer_config_helpers/tests/configs/file_list.sh

@@ -5,6 +5,6 @@ last_first_seq test_expand_layer test_ntm_layers test_hsigmoid
 img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cost_layers
 test_rnn_group shared_fc shared_lstm shared_gru test_cost_layers_with_weight
 test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops
-test_seq_concat_reshape test_pad test_smooth_l1)
+test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer)
 
 export whole_configs=(test_split_datasource)

python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py

@@ -6,7 +6,7 @@ din = data_layer(name='data', size=30)
 
 seq_op = [first_seq, last_seq]
 
-agg_level = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP]
+agg_level = [AggregateLevel.TO_SEQUENCE, AggregateLevel.TO_NO_SEQUENCE]
 
 opts = []
 
@@ -15,6 +15,7 @@ for op in seq_op:
         opts.append(op(input=din, agg_level=al))
 
 for op in seq_op:
-    opts.append(op(input=din, agg_level=AggregateLevel.EACH_TIMESTEP, stride=5))
+    opts.append(
+        op(input=din, agg_level=AggregateLevel.TO_NO_SEQUENCE, stride=5))
 
 outputs(opts)

python/paddle/trainer_config_helpers/tests/configs/protostr/test_multiplex_layer.protostr

+type: "nn"
+layers {
+  name: "index"
+  type: "data"
+  size: 1
+  active_type: ""
+}
+layers {
+  name: "data1"
+  type: "data"
+  size: 30
+  active_type: ""
+}
+layers {
+  name: "data2"
+  type: "data"
+  size: 30
+  active_type: ""
+}
+layers {
+  name: "data3"
+  type: "data"
+  size: 30
+  active_type: ""
+}
+layers {
+  name: "__multiplex_layer_0__"
+  type: "multiplex"
+  size: 30
+  active_type: ""
+  inputs {
+    input_layer_name: "index"
+  }
+  inputs {
+    input_layer_name: "data1"
+  }
+  inputs {
+    input_layer_name: "data2"
+  }
+  inputs {
+    input_layer_name: "data3"
+  }
+}
+input_layer_names: "index"
+input_layer_names: "data1"
+input_layer_names: "data2"
+input_layer_names: "data3"
+output_layer_names: "__multiplex_layer_0__"
+sub_models {
+  name: "root"
+  layer_names: "index"
+  layer_names: "data1"
+  layer_names: "data2"
+  layer_names: "data3"
+  layer_names: "__multiplex_layer_0__"
+  input_layer_names: "index"
+  input_layer_names: "data1"
+  input_layer_names: "data2"
+  input_layer_names: "data3"
+  output_layer_names: "__multiplex_layer_0__"
+  is_recurrent_layer_group: false
+}
+

python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py

@@ -9,4 +9,6 @@ outputs(
     expand_layer(
         input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_SEQUENCE),
     expand_layer(
-        input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_TIMESTEP))
+        input=din,
+        expand_as=data_seq,
+        expand_level=ExpandLevel.FROM_NO_SEQUENCE))

python/paddle/trainer_config_helpers/tests/configs/test_multiplex_layer.py

+from paddle.trainer_config_helpers import *
+
+settings(batch_size=1000, learning_rate=1e-5)
+
+index = data_layer(name='index', size=1)
+din1 = data_layer(name='data1', size=30)
+din2 = data_layer(name='data2', size=30)
+din3 = data_layer(name='data3', size=30)
+
+dout = multiplex_layer([index, din1, din2, din3])
+
+outputs(dout)

python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py

@@ -6,7 +6,7 @@ din = data_layer(name='dat_in', size=100)
 
 POOL_TYPE = [MaxPooling, AvgPooling, SumPooling]
 
-AGG_LEVEL = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP]
+AGG_LEVEL = [AggregateLevel.TO_SEQUENCE, AggregateLevel.TO_NO_SEQUENCE]
 
 opts = []
 

python/paddle/utils/make_model_diagram.py

@@ -39,6 +39,10 @@ def make_layer_label(layer_config):
 
 def make_diagram(config_file, dot_file, config_arg_str):
     config = parse_config(config_file, config_arg_str)
+    make_diagram_from_proto(config.model_config, dot_file)
+
+
+def make_diagram_from_proto(model_config, dot_file):
     # print >> sys.stderr, config
     name2id = {}
     f = open(dot_file, 'w')
@@ -59,12 +63,12 @@ def make_diagram(config_file, dot_file, config_arg_str):
 
     print >> f, 'digraph graphname {'
     print >> f, 'node [width=0.375,height=0.25];'
-    for i in xrange(len(config.model_config.layers)):
-        l = config.model_config.layers[i]
+    for i in xrange(len(model_config.layers)):
+        l = model_config.layers[i]
         name2id[l.name] = i
 
     i = 0
-    for sub_model in config.model_config.sub_models:
+    for sub_model in model_config.sub_models:
         if sub_model.name == 'root':
             continue
         print >> f, 'subgraph cluster_%s {' % i
@@ -78,18 +82,18 @@ def make_diagram(config_file, dot_file, config_arg_str):
         for layer_name in sub_model.layer_names:
             submodel_layers.add(layer_name)
             lid = name2id[layer_name]
-            layer_config = config.model_config.layers[lid]
+            layer_config = model_config.layers[lid]
             label = make_layer_label(layer_config)
             print >> f, 'l%s [label="%s", shape=box];' % (lid, label)
         print >> f, '}'
 
-    for i in xrange(len(config.model_config.layers)):
-        l = config.model_config.layers[i]
+    for i in xrange(len(model_config.layers)):
+        l = model_config.layers[i]
         if l.name not in submodel_layers:
             label = make_layer_label(l)
             print >> f, 'l%s [label="%s", shape=box];' % (i, label)
 
-    for sub_model in config.model_config.sub_models:
+    for sub_model in model_config.sub_models:
         if sub_model.name == 'root':
             continue
         for link in sub_model.in_links:
@@ -99,8 +103,8 @@ def make_diagram(config_file, dot_file, config_arg_str):
         for mem in sub_model.memories:
             print >> f, make_mem(mem)
 
-    for i in xrange(len(config.model_config.layers)):
-        for l in config.model_config.layers[i].inputs:
+    for i in xrange(len(model_config.layers)):
+        for l in model_config.layers[i].inputs:
             print >> f, 'l%s -> l%s [label="%s"];' % (
                 name2id[l.input_layer_name], i, l.input_parameter_name)
 

python/paddle/v2/__init__.py

@@ -27,6 +27,7 @@ from . import dataset
 from . import reader
 from . import plot
 import attr
+import op
 import pooling
 import inference
 import networks

python/paddle/v2/config_base.py

@@ -14,206 +14,55 @@
 
 import collections
 import re
-from paddle.trainer_config_helpers.default_decorators import wrap_name_default
 import paddle.trainer_config_helpers as conf_helps
-from topology import Topology
-
-
-class LayerType(type):
-    def __new__(cls, name, bases, attrs):
-        method_name = attrs.get('METHOD_NAME', None)
-        if method_name is not None:
-            method = getattr(conf_helps, method_name)
-            if method.__doc__ is not None:
-                mapper = attrs.get("__map_docstr__", None)
-                if mapper is not None:
-                    attrs['__doc__'] = LayerType.__map_docstr__(
-                        mapper(method.__doc__),
-                        method_name=method_name,
-                        name=name)
-                else:
-                    attrs['__doc__'] = LayerType.__map_docstr__(
-                        method.__doc__, method_name=method_name, name=name)
-        return super(LayerType, cls).__new__(cls, name, bases, attrs)
-
-    @staticmethod
-    def __map_docstr__(doc, name, method_name):
-        assert isinstance(doc, basestring)
-
-        # replace LayerOutput to paddle.v2.config_base.Layer
-        doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer")
-
-        doc = doc.replace('ParameterAttribute',
-                          'paddle.v2.attr.ParameterAttribute')
-
-        doc = re.sub(r'ExtraLayerAttribute[^\s]?',
-                     'paddle.v2.attr.ExtraAttribute', doc)
-
-        # xxx_layer to xxx
-        doc = re.sub(r"(?P<name>[a-z]+)_layer", r"\g<name>", doc)
-
-        # XxxxActivation to paddle.v2.Activation.Xxxx
-        doc = re.sub(r"(?P<name>[A-Z][a-zA-Z]+)Activation",
-                     r"paddle.v2.Activation.\g<name>", doc)
-
-        # TODO(yuyang18): Add more rules if needed.
+
+__layer_map__ = {}
+
+
+def __map_docstr__(doc, name):
+    if doc is None:
         return doc
 
+    assert isinstance(doc, basestring)
+
+    # replace LayerOutput to paddle.v2.config_base.Layer
+    doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer")
+
+    doc = doc.replace('ParameterAttribute', 'paddle.v2.attr.ParameterAttribute')
+
+    doc = re.sub(r'ExtraLayerAttribute[^\s]?', 'paddle.v2.attr.ExtraAttribute',
+                 doc)
+
+    # xxx_layer to xxx
+    doc = re.sub(r"(?P<name>[a-z]+)_layer", r"\g<name>", doc)
+
+    # XxxxActivation to paddle.v2.activation.Xxxx
+    doc = re.sub(r"(?P<name>[A-Z][a-zA-Z]+)Activation",
+                 r"paddle.v2.activation.\g<name>", doc)
+
+    # xxx_evaluator to paddle.v2.evaluator.xxx
+    doc = re.sub(r"(?P<name>[a-z]+)_evaluator", r"evaluator.\g<name>", doc)
+
+    # TODO(yuyang18): Add more rules if needed.
+    return doc
+
+
+def __convert_to_v2__(f, name, module):
+    def wrapped(*args, **xargs):
+        out = f(*args, **xargs)
+        outs = out
+        if not isinstance(out, collections.Sequence):
+            outs = [out]
+        for l in outs:
+            if isinstance(l, conf_helps.LayerOutput):
+                __layer_map__[l.full_name] = l
+        return out
+
+    wrapped.__doc__ = __map_docstr__(f.__doc__, name)
+    wrapped.__name__ = name
+    wrapped.__module__ = module
+
+    return wrapped
+
 
-class Layer(object):
-    __metaclass__ = LayerType
-
-    def __init__(self, name=None, parent_layers=None):
-        assert isinstance(parent_layers, dict)
-        self.name = name
-        self.__context__ = {}
-        self.__parent_layers__ = parent_layers
-        # some layer may have some extra parent layer
-        self.__extra_parent__ = []
-        # used for evaluator.
-        self.__children_layers__ = []
-
-    def extra_parent(self):
-        return self.__extra_parent__
-
-    def append_extra_parent(self, parent):
-        self.__extra_parent__.append(parent)
-
-    def append_child(self, layer, parent_names):
-        self.__children_layers__.append((layer, parent_names))
-
-    def to_proto(self, context):
-        """
-        function to set proto attribute
-        """
-        self.__context__ = context
-
-        # STEP: short cut if this layer is parsed before.
-        if self.context_name() in context:
-            if self.use_context_name():
-                return context[self.context_name()]
-            else:
-                return context[self.name]
-
-        # STEP: parse extra_parent that is not used by this layer but must
-        # be parsed before this layer.
-        for p in self.__extra_parent__:
-            p.to_proto(context=context)
-
-        # STEP: parse parent that is used by this layer, get the result and
-        # insert into kwargs of the next layer's to_proto_impl method.
-        kwargs = dict()
-        for layer_name in self.__parent_layers__:
-            if not isinstance(self.__parent_layers__[layer_name],
-                              collections.Sequence):
-                v1_layer = self.__parent_layers__[layer_name].to_proto(
-                    context=context)
-            else:
-                v1_layer = map(lambda x: x.to_proto(context=context),
-                               self.__parent_layers__[layer_name])
-            kwargs[layer_name] = v1_layer
-
-        # STEP: parse myself and add myself into context.
-        ret_val = self.to_proto_impl(**kwargs)
-        if self.context_name() is not None \
-                and self.context_name() not in context:
-            context[self.context_name()] = ret_val
-
-        # STEP: parse children that should be pased after this layer.
-        for layer, pnames in self.__children_layers__:
-            drop = False
-
-            # child will only be parsed if all parents are in context.
-            for pname in pnames:
-                if pname not in context:
-                    drop = True
-                    break
-            if drop:
-                continue
-            layer.to_proto(context=context)
-
-        # STEP: return v1 layer result
-        if self.context_name() is None:
-            return ret_val
-        elif self.use_context_name():
-            return context[self.context_name()]
-        else:
-            return context[self.name]
-
-    def to_proto_impl(self, **kwargs):
-        raise NotImplementedError()
-
-    def context_name(self):
-        """
-        Context name means the context which stores `to_proto_impl` result.
-        If multiple layer share same context_name, the `to_proto_impl` of them
-        will be invoked only once.
-        """
-        return self.name
-
-    def use_context_name(self):
-        return False
-
-    def calculate_size(self):
-        """
-        lazy calculate size of the layer, should be called when to_proto_impl of
-        this layer is called.
-        :return:
-        """
-        return self.__context__[self.context_name()].size
-
-    def attr(self):
-        topo = Topology(self)
-        return topo.get_layer_proto(self.name)
-
-
-def __convert_to_v2__(method_name,
-                      parent_names,
-                      is_default_name=True,
-                      attach_parent=False):
-    if is_default_name:
-        wrapper = wrap_name_default(name_prefix=method_name)
-    else:
-        wrapper = None
-
-    class V2LayerImpl(Layer):
-        METHOD_NAME = method_name
-
-        def __init__(self, **kwargs):
-            parent_layers = dict()
-            other_kwargs = dict()
-            for pname in parent_names:
-                if pname in kwargs:
-                    parent_layers[pname] = kwargs[pname]
-
-            if attach_parent:
-                pnames = [x.context_name() for x in parent_layers.values()]
-
-                for pname in parent_layers:
-                    layers = kwargs[pname]
-                    if not isinstance(layers, collections.Sequence):
-                        layers = [layers]
-
-                    for layer in layers:
-                        layer.append_child(self, pnames)
-
-            for key in kwargs.keys():
-                if key not in parent_names:
-                    other_kwargs[key] = kwargs[key]
-
-            name = kwargs.get('name', None)
-            super(V2LayerImpl, self).__init__(name, parent_layers)
-            self.__other_kwargs__ = other_kwargs
-
-        if wrapper is not None:
-            __init__ = wrapper(__init__)
-
-        def to_proto_impl(self, **kwargs):
-            args = dict()
-            for each in kwargs:
-                args[each] = kwargs[each]
-            for each in self.__other_kwargs__:
-                args[each] = self.__other_kwargs__[each]
-            return getattr(conf_helps, method_name)(**args)
-
-    return V2LayerImpl
+Layer = conf_helps.LayerOutput

python/paddle/v2/evaluator.py

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 import paddle.trainer_config_helpers.evaluators as evs
-import inspect
 from config_base import __convert_to_v2__
+import inspect
 
 __all__ = []
 
@@ -25,21 +25,10 @@ def initialize():
 
     for __ev_name__ in filter(lambda x: x.endswith('_evaluator'), evs.__all__):
         __ev__ = getattr(evs, __ev_name__)
-        if hasattr(__ev__, 'argspec'):
-            argspec = __ev__.argspec
-        else:
-            argspec = inspect.getargspec(__ev__)
-        parent_names = filter(lambda x: x in ['input', 'label', 'weight'],
-                              argspec.args)
-        v2_ev = __convert_to_v2__(
-            __ev_name__,
-            parent_names=parent_names,
-            is_default_name='name' in argspec.args,
-            attach_parent=True)
-
         __new_name__ = convert_to_new_name(__ev_name__)
 
-        globals()[__new_name__] = v2_ev
+        globals()[__new_name__] = __convert_to_v2__(__ev__, __new_name__,
+                                                    __name__)
         globals()[__new_name__].__name__ = __new_name__
         __all__.append(__new_name__)
 

python/paddle/v2/inference.py

@@ -12,9 +12,9 @@ class Inference(object):
     """
     Inference combines neural network output and parameters together
     to do inference.
-    
+
     ..  code-block:: python
-    
+
         inferer = Inference(output_layer=prediction, parameters=parameters)
         for data_batch in batches:
             print inferer.infer(data_batch)
@@ -92,8 +92,8 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
 
     ..  code-block:: python
 
-        result = paddle.infer(output_layer=prediction, 
-                              parameters=parameters, 
+        result = paddle.infer(output_layer=prediction,
+                              parameters=parameters,
                               input=SomeData)
         print result
 
@@ -101,14 +101,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
 
     ..  code-block:: python
 
-        result = paddle.infer(output_layer=[prediction1, prediction2], 
-                              parameters=parameters, 
+        result = paddle.infer(output_layer=[prediction1, prediction2],
+                              parameters=parameters,
                               input=SomeData,
                               field=[id, value]])
         print result
 
     :param output_layer: output of the neural network that would be inferred
-    :type output_layer: paddle.v2.config_base.Layer or a list of 
+    :type output_layer: paddle.v2.config_base.Layer or a list of
                         paddle.v2.config_base.Layer
     :param parameters: parameters of the neural network.
     :type parameters: paddle.v2.parameters.Parameters
@@ -117,14 +117,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
     :type input: collections.Iterable
     :param feeding: Reader dictionary. Default could generate from input
                         value.
-    :param field: The prediction field. It should in [`value`, `id`, `prob`]. 
-                  `value` and `prob` mean return the prediction probabilities, 
+    :param field: The prediction field. It should in [`value`, `id`, `prob`].
+                  `value` and `prob` mean return the prediction probabilities,
                   `id` means return the prediction labels. Default is `value`.
-                  Note that `prob` only used when output_layer is beam_search 
+                  Note that `prob` only used when output_layer is beam_search
                   or max_id.
     :type field: str
-    :return: The prediction result. If there are multiple outout_layers and fields, 
-             the return order is outout_layer1.field1, outout_layer2.field1, ..., 
+    :return: The prediction result. If there are multiple outout_layers and fields,
+             the return order is outout_layer1.field1, outout_layer2.field1, ...,
              outout_layer1.field2, outout_layer2.field2 ...
     :rtype: numpy.ndarray
     """

python/paddle/v2/layer.py

@@ -32,392 +32,29 @@ The primary usage shows below.
 """
 
 import collections
-import inspect
+import copy
 import re
+import paddle.trainer_config_helpers.layers as v1_layers
+import paddle.trainer.config_parser as cp
+from paddle.proto.ModelConfig_pb2 import ModelConfig, SubModelConfig
+from config_base import __convert_to_v2__
+import config_base
 
-import paddle.trainer_config_helpers as conf_helps
-from paddle.trainer.config_parser import \
-    RecurrentLayerGroupWithoutOutLinksBegin, RecurrentLayerGroupSetOutLink, \
-    RecurrentLayerGroupEnd, model_type
-from paddle.trainer_config_helpers.config_parser_utils import \
-    parse_network_config as __parse__
-from paddle.trainer_config_helpers.default_decorators import wrap_act_default
-from paddle.trainer_config_helpers.default_decorators import \
-    wrap_bias_attr_default
-from paddle.trainer_config_helpers.default_decorators import wrap_name_default
-from paddle.trainer_config_helpers.layers import RecurrentLayerGroupSetGenerator, Generator
-from paddle.trainer_config_helpers.layers import layer_support
+__all__ = ['data', 'parse_network']
 
-import activation
-import attr
-import data_type
-from config_base import Layer, __convert_to_v2__
 
-__all__ = ['parse_network', 'data']
+def __need_to_keep__(name):
+    if name in ['StaticInput', 'LayerType', 'layer_support']:
+        return False
+    return True
 
 
-def parse_network(output_layers, extra_layers=None):
-    """
-    Parse all layers in the neural network graph and
-    then generate a ModelConfig object.
-
-    ..  note::
-
-        This function is used internally in paddle.v2 module. User should never
-        invoke this method.
-
-    :param output_layers: Output layers.
-    :type output_layers: Layer
-    :param extra_layers: Some layers in the neural network graph are not in the
-                         path of output_layers.
-    :type extra_layers: Layer
-    :return: A ModelConfig object instance.
-    :rtype: ModelConfig
-    """
-    if not isinstance(output_layers, collections.Sequence):
-        output_layers = [output_layers]
-    if extra_layers is not None and not isinstance(extra_layers,
-                                                   collections.Sequence):
-        extra_layers = [extra_layers]
-
-    def __real_func__():
-        """
-        __real_func__ is the function that config_parser.parse invoked. It is
-        the plain old paddle configuration function.
-        """
-        context = dict()
-        real_output = [each.to_proto(context=context) for each in output_layers]
-        if extra_layers is not None:
-            extra_output = [
-                each.to_proto(context=context) for each in extra_layers
-            ]
-        conf_helps.outputs(real_output)
+def __need_to_wrap__(name):
+    return name not in ['AggregateLevel', 'ExpandLevel']
 
-    return __parse__(__real_func__)
 
-
-"""
-Some layer may need some special config, and can not use __convert_to_v2__ to convert.
-So we also need to implement some special LayerV2.
-"""
-
-
-class DataLayerV2(Layer):
-    METHOD_NAME = 'data_layer'
-
-    def __init__(self, name, type, **kwargs):
-        assert isinstance(type, data_type.InputType)
-
-        self.type = type
-        self.__method_name__ = 'data_layer'
-        self.__kwargs__ = kwargs
-
-        super(DataLayerV2, self).__init__(name=name, parent_layers=dict())
-
-    def to_proto_impl(self, **kwargs):
-        args = dict()
-        args['size'] = self.type.dim
-        for each in kwargs:
-            args[each] = kwargs[each]
-        for each in self.__kwargs__:
-            args[each] = self.__kwargs__[each]
-        return getattr(conf_helps, self.__method_name__)(name=self.name, **args)
-
-    def __map_docstr__(doc):
-        doc = re.sub(r'(data = [^\)]+)\).*',
-                     "data = paddle.layer.data(name=\"input\", "
-                     "type=paddle.data_type.dense_vector(1000))", doc)
-
-        doc = re.sub(r':param size:.*',
-                     ':param type: Data type of this data layer', doc)
-        doc = re.sub(r':type size:.*',
-                     ":type size: paddle.v2.data_type.InputType", doc)
-        return doc
-
-
-class MemoryV2(Layer):
-    def __init__(self, name, extra_input=None, **kwargs):
-        """
-        Init memory object, if memory is inited inside recurrent_group step
-        function, it may depend on a boot_layer that should be initialized
-        outside recurrent_group, so we:
-            1. add RecurrentLayerInput to extra_parent of self.
-            2. add boot_layer to the extra_parent of RecurrentLayerInput.
-
-        :param extra_input: list of RecurrentLayerInput
-        :type extra_input: [RecurrentLayerInput]
-        """
-        self.name = name
-        super(MemoryV2, self).__init__(name=name, parent_layers=dict())
-        self.__kwargs__ = kwargs
-        self.__boot_layer_name__ = None
-
-        if 'boot_layer' in kwargs:
-            begin_of_current_rnn = []
-            # TODO(yuyang18): Fix inspect, it could be wrong when user invoke a
-            # function inside step.
-            st = inspect.stack()
-            for i in xrange(len(st)):
-                locs = inspect.stack()[i][0].f_locals
-                keys = locs.keys()
-                for key in keys:
-                    val = locs[key]
-                    if isinstance(val, RecurrentLayerInput):
-                        begin_of_current_rnn.append(val)
-                    elif isinstance(val, collections.Sequence):
-                        for v in val:
-                            if isinstance(v, RecurrentLayerInput):
-                                begin_of_current_rnn.append(v)
-
-                if begin_of_current_rnn:
-                    break
-            assert begin_of_current_rnn is not None
-            for extra in begin_of_current_rnn:
-                self.append_extra_parent(extra)
-                extra.append_extra_parent(kwargs['boot_layer'])
-                self.__boot_layer_name__ = kwargs['boot_layer'].name
-
-    def to_proto_impl(self, **kwargs):
-        args = dict()
-        for each in kwargs:
-            args[each] = kwargs[each]
-        for each in self.__kwargs__:
-            args[each] = self.__kwargs__[each]
-
-        if self.__boot_layer_name__ is not None:
-            args['boot_layer'] = self.__context__[self.__boot_layer_name__]
-
-        size = args.get('size', None)
-        if size is not None:
-            if callable(size):
-                real_size = size()
-            else:
-                real_size = size
-            args['size'] = real_size
-        return conf_helps.memory(name=self.name, **args)
-
-    def context_name(self):
-        return self.name + "#memory"
-
-    def use_context_name(self):
-        """
-        memory layer will have the same name with some layer
-        :return:
-        """
-        return True
-
-
-class StaticInputV2(object):
-    def __init__(self, input, is_seq=False, size=None):
-        assert isinstance(input, LayerV2)
-        self.name = input.name
-        self.input = input
-        self.is_seq = is_seq
-        self.size = size
-        # TODO(add size check)
-        # assert input.size is not None or size is not None
-
-
-class BaseGeneratedInputV2(object):
-    def __init__(self):
-        self.bos_id = None
-        self.eos_id = None
-
-    def before_real_step(self):
-        raise NotImplementedError()
-
-    def after_real_step(self, *args):
-        raise NotImplementedError()
-
-
-class GeneratedInputV2(BaseGeneratedInputV2):
-    def __init__(self, size, embedding_name, embedding_size):
-        super(GeneratedInputV2, self).__init__()
-        self.size = size
-        self.embedding_name = embedding_name
-        self.embedding_size = embedding_size
-
-    def after_real_step(self, input):
-        return max_id(input=input, name='__beam_search_predict__')
-
-    def before_real_step(self):
-        predict_id = memory(
-            name='__beam_search_predict__',
-            size=self.size,
-            boot_with_const_id=self.bos_id)
-
-        trg_emb = embedding(
-            input=predict_id,
-            size=self.embedding_size,
-            param_attr=attr.ParamAttr(name=self.embedding_name))
-        return trg_emb
-
-
-class RecurrentLayerGroupSetGeneratorV2(Layer):
-    def __init__(self, eos_name, max_length, beam_size, num_results_per_sample):
-        self.eos_name = eos_name
-        self.max_length = max_length
-        self.beam_size = beam_size
-        self.num_results_per_sample = num_results_per_sample
-        super(RecurrentLayerGroupSetGeneratorV2, self).__init__(
-            name=eos_name, parent_layers={})
-
-    def to_proto_impl(self, **kwargs):
-        RecurrentLayerGroupSetGenerator(
-            Generator(
-                eos_layer_name=self.eos_name,
-                max_num_frames=self.max_length,
-                beam_size=self.beam_size,
-                num_results_per_sample=self.num_results_per_sample))
-        return self
-
-    def context_name(self):
-        return self.eos_name + ".fake"
-
-    def use_context_name(self):
-        return True
-
-
-class MixedLayerV2(Layer):
-    """
-    This class is use to support `with` grammar. If not, the following code
-    could convert mixed_layer simply.
-
-        mixed = __convert_to_v2__(
-            'mixed_layer', name_prefix='mixed', parent_names=['input'])
-    """
-
-    class AddToSealedMixedLayerExceptionV2(Exception):
-        pass
-
-    def __init__(self,
-                 size=0,
-                 input=None,
-                 name=None,
-                 act=None,
-                 bias_attr=None,
-                 layer_attr=None):
-        self.__method_name__ = 'mixed_layer'
-        self.finalized = False
-        self.__inputs__ = []
-        if input is not None:
-            self.__inputs__ = input
-
-        other_kwargs = dict()
-        other_kwargs['name'] = name
-        other_kwargs['size'] = size
-        other_kwargs['act'] = act
-        other_kwargs['bias_attr'] = bias_attr
-        other_kwargs['layer_attr'] = layer_attr
-        parent_layers = {"input": self.__inputs__}
-        super(MixedLayerV2, self).__init__(name, parent_layers)
-        self.__other_kwargs__ = other_kwargs
-
-    def __iadd__(self, other):
-        if not self.finalized:
-            self.__inputs__.append(other)
-            return self
-        else:
-            raise MixedLayerV2.AddToSealedMixedLayerExceptionV2()
-
-    def __enter__(self):
-        assert len(self.__inputs__) == 0
-        return self
-
-    def __exit__(self, *args, **kwargs):
-        self.finalized = True
-
-    def to_proto_impl(self, **kwargs):
-        args = dict()
-        for each in kwargs:
-            args[each] = kwargs[each]
-        for each in self.__other_kwargs__:
-            args[each] = self.__other_kwargs__[each]
-        size = args.get('size', None)
-        if size is not None:
-            if callable(size):
-                real_size = size()
-            else:
-                real_size = size
-            args['size'] = real_size
-        return getattr(conf_helps, self.__method_name__)(**args)
-
-
-@wrap_name_default("mixed")
-@wrap_act_default(act=activation.Linear())
-@wrap_bias_attr_default(has_bias=False)
-@layer_support(conf_helps.layers.ERROR_CLIPPING, conf_helps.layers.DROPOUT)
-def mixed(size=0,
-          name=None,
-          input=None,
-          act=None,
-          bias_attr=False,
-          layer_attr=None):
-    return MixedLayerV2(size, input, name, act, bias_attr, layer_attr)
-
-
-mixed.__doc__ = conf_helps.mixed_layer.__doc__
-
-
-class RecurrentLayerInput(Layer):
-    def __init__(self, recurrent_name, index, parent_layers, reverse):
-        parents_len = len(parent_layers)
-        assert parents_len <= 1
-        if parents_len == 0:
-            self.__parents__ = []
-        else:
-            self.__parents__ = parent_layers.values()[0]
-        self.__recurrent_name__ = recurrent_name
-        self.__reverse__ = reverse
-        name = self.__parents__[
-            index].name if index >= 0 else self.context_name()
-        super(RecurrentLayerInput, self).__init__(
-            name=name, parent_layers=parent_layers)
-
-    def context_name(self):
-        return self.__recurrent_name__ + ".begin"
-
-    def to_proto_impl(self, **kwargs):
-        model_type('recurrent_nn')
-        RecurrentLayerGroupWithoutOutLinksBegin(
-            name=self.__recurrent_name__,
-            in_links=map(lambda x: x.name, self.__parents__),
-            seq_reversed=self.__reverse__)
-        return self
-
-
-class RecurrentLayerOutput(Layer):
-    def __init__(self, recurrent_name, index, parent_layers):
-        assert len(parent_layers) == 1
-        self.__parents__ = parent_layers.values()[0]
-        super(RecurrentLayerOutput, self).__init__(
-            name=self.__parents__[index].name, parent_layers=parent_layers)
-        self.__recurrent_name__ = recurrent_name
-
-    def context_name(self):
-        return self.__recurrent_name__ + ".end"
-
-    def to_proto_impl(self, **kwargs):
-        for l in self.__parents__:
-            RecurrentLayerGroupSetOutLink(l.name)
-        RecurrentLayerGroupEnd(name=self.__recurrent_name__)
-
-
-LayerV2 = Layer
-data = DataLayerV2
-data.__name__ = 'data'
-AggregateLevel = conf_helps.AggregateLevel
-ExpandLevel = conf_helps.ExpandLevel
-memory = MemoryV2
-memory.__name__ = 'memory'
-memory.__doc__ = conf_helps.memory.__doc__
-
-
-def __layer_name_mapping__(inname):
-    if inname in ['data_layer', 'memory', 'mixed_layer', 'recurrent_group']:
-        # Do Not handle these layers
-        return
-    elif inname == 'maxid_layer':
+def __convert_name__(inname):
+    if inname == 'maxid_layer':
         return 'max_id'
     elif inname.endswith('memory') or inname.endswith(
             '_seq') or inname.endswith('_sim') or inname == 'hsigmoid':
@@ -431,187 +68,220 @@ def __layer_name_mapping__(inname):
         return inname
     elif inname.endswith("_layer"):
         return inname[:-len("_layer")]
+    else:
+        return inname
 
 
-def __layer_name_mapping_parent_names__(inname):
-    all_args = getattr(conf_helps, inname).argspec.args
-    return filter(
-        lambda x: x in ['input1', 'input2', 'label', 'input', 'a', 'b',
-                        'expand_as',
-                        'weights', 'vectors', 'weight', 'score', 'left',
-                        'right', 'output_mem'],
-        all_args)
-
-
-def __convert_layer__(_new_name_, _old_name_, _parent_names_):
-    global __all__
-    __all__.append(_new_name_)
-    globals()[new_name] = __convert_to_v2__(_old_name_, _parent_names_)
-    globals()[new_name].__name__ = new_name
-
-
-for each_layer_name in dir(conf_helps):
-    new_name = __layer_name_mapping__(each_layer_name)
-    if new_name is not None:
-        parent_names = __layer_name_mapping_parent_names__(each_layer_name)
-        assert len(parent_names) != 0, each_layer_name
-        __convert_layer__(new_name, each_layer_name, parent_names)
-
-del parent_names
-del new_name
-del each_layer_name
-
-
-@wrap_name_default()
-def recurrent_group(step, input, reverse=False, name=None):
-    if not isinstance(input, collections.Sequence):
-        input = [input]
-
-    non_static_inputs = filter(lambda x: not isinstance(x, StaticInputV2),
-                               input)
-    actual_input = [
-        RecurrentLayerInput(
-            recurrent_name=name,
-            index=i,
-            parent_layers={'recurrent_inputs': non_static_inputs},
-            reverse=reverse) for i in xrange(len(non_static_inputs))
-    ]
-
-    extra_input = None
-    if len(non_static_inputs) == 0:
-        extra_input = RecurrentLayerInput(
-            recurrent_name=name, index=-1, parent_layers={}, reverse=reverse)
-
-    def __real_step__(*args):
-        rnn_input = list(args)
-        static_inputs = filter(lambda x: isinstance(x, StaticInputV2), input)
-        for static_input in static_inputs:
-            mem_name = "__%s_memory__" % static_input.input.name
-            mem = memory(
-                name=mem_name,
-                extra_input=extra_input,
-                is_seq=static_input.is_seq,
-                size=static_input.input.calculate_size,
-                boot_layer=static_input.input)
-            with mixed(
-                    name=mem_name,
-                    size=static_input.input.calculate_size,
-                    act=activation.Identity()) as mix:
-                mix += identity_projection(input=mem)
-            rnn_input.insert(input.index(static_input), mix)
-        return step(*rnn_input)
-
-    actual_output = __real_step__(*actual_input)
-
-    if not isinstance(actual_output, collections.Sequence):
-        actual_output = [actual_output]
-
-    retv = [
-        RecurrentLayerOutput(
-            recurrent_name=name,
-            index=i,
-            parent_layers={'recurrent_outputs': actual_output})
-        for i in xrange(len(actual_output))
-    ]
-    if len(retv) == 1:
-        return retv[0]
+for name in v1_layers.__all__:
+    obj = getattr(v1_layers, name)
+    if not __need_to_keep__(name):
+        continue
+    new_name = __convert_name__(name)
+    if callable(obj) and __need_to_wrap__(name):
+        globals()[new_name] = __convert_to_v2__(obj, new_name, __name__)
     else:
-        return retv
-
-
-recurrent_group.__doc__ = conf_helps.recurrent_group.__doc__
-
-
-@wrap_name_default()
-def beam_search(step,
-                input,
-                bos_id,
-                eos_id,
-                beam_size,
-                max_length=500,
-                name=None,
-                num_results_per_sample=None):
-    if num_results_per_sample is None:
-        num_results_per_sample = beam_size
-    assert num_results_per_sample <= beam_size
-    # logger.warning("num_results_per_sample should be less than beam_size")
-
-    if isinstance(input, StaticInputV2) or isinstance(input,
-                                                      BaseGeneratedInputV2):
-        input = [input]
-
-    generated_input_index = -1
-
-    real_input = []
-    for i, each_input in enumerate(input):
-        assert isinstance(each_input, StaticInputV2) or isinstance(
-            each_input, BaseGeneratedInputV2)
-        if isinstance(each_input, BaseGeneratedInputV2):
-            assert generated_input_index == -1
-            generated_input_index = i
-        else:
-            real_input.append(each_input)
+        globals()[new_name] = obj
+    __all__.append(new_name)
 
-    assert generated_input_index != -1
 
-    gipt = input[generated_input_index]
-    assert isinstance(gipt, BaseGeneratedInputV2)
+def __data_layer__(name, type, **kwargs):
+    l = v1_layers.data_layer(name, type.dim, **kwargs)
+    l.data_type = type
+    return l
 
-    gipt.bos_id = bos_id
-    gipt.eos_id = eos_id
 
-    def __real_step__(*args):
-        eos_name = "__%s_eos_layer__" % name
-        generator = RecurrentLayerGroupSetGeneratorV2(
-            eos_name, max_length, beam_size, num_results_per_sample)
+def __map_data_docstr__(doc):
+    doc = re.sub(r'(data = [^\)]+)\).*',
+                 "data = paddle.layer.data(name=\"input\", "
+                 "type=paddle.data_type.dense_vector(1000))", doc)
 
-        args = list(args)
-        before_step_layer = gipt.before_real_step()
-        before_step_layer.append_child(
-            layer=generator, parent_names=[before_step_layer.name])
-        args.insert(generated_input_index, before_step_layer)
+    doc = re.sub(r':param size:.*', ':param type: Data type of this data layer',
+                 doc)
+    doc = re.sub(r':type size:.*', ":type size: paddle.v2.data_type.InputType",
+                 doc)
+    return doc
+
+
+__data_layer__.__doc__ = __map_data_docstr__(v1_layers.data_layer.__doc__)
+
+data = __convert_to_v2__(__data_layer__, 'name', __name__)
+
+
+def __get_used_layers__(output_layers, extra_layers=None):
+    layer_names = set()
+    parents = {}
+
+    def add_parent(child, parent):
+        if child in parents:
+            parents[child].append(parent)
+        else:
+            parents[child] = [parent]
+
+    def add_additional_parents():
+        for sub_model in cp.g_config.model_config.sub_models:
+            if sub_model.name == 'root':
+                continue
+            for link in sub_model.in_links:
+                add_parent(link.link_name, link.layer_name)
+                add_parent(sub_model.name, link.layer_name)
+            for link in sub_model.out_links:
+                add_parent(link.link_name, link.layer_name)
+                add_parent(link.link_name, sub_model.name)
+            for mem in sub_model.memories:
+                if mem.boot_layer_name:
+                    add_parent(mem.layer_name, mem.boot_layer_name)
+                add_parent(mem.link_name, mem.layer_name)
+
+    def dfs_travel(layer_name):
+        if layer_name in layer_names:
+            return
+        layer_names.add(layer_name)
+        layer = cp.g_layer_map[layer_name]
+
+        for inp in layer.inputs:
+            dfs_travel(inp.input_layer_name)
+        if layer.name in parents:
+            for p in parents[layer.name]:
+                dfs_travel(p)
+
+    add_additional_parents()
+
+    for layer in output_layers:
+        dfs_travel(layer.full_name)
+
+    return layer_names
+
+
+def __get_used_parameters__(layer_names, sub_models):
+    parameter_names = set()
+    for name in layer_names:
+        l = cp.g_layer_map[name]
+        for inp in l.inputs:
+            if inp.input_parameter_name:
+                parameter_names.add(inp.input_parameter_name)
+        if l.bias_parameter_name:
+            parameter_names.add(l.bias_parameter_name)
+
+    for sub_model in sub_models:
+        for mem in sub_model.memories:
+            if mem.HasField("boot_bias_parameter_name"):
+                parameter_names.add(mem.boot_bias_parameter_name)
+
+    return parameter_names
+
+
+def __get_used_submodels__(layer_names):
+    submodel_names = set()
+    for submodel in cp.g_config.model_config.sub_models:
+        if submodel.name in layer_names:
+            submodel_names.add(submodel.name)
+    return submodel_names
+
+
+def __get_used_evaluators__(layer_names):
+    evaluator_names = set()
+    for e in cp.g_config.model_config.evaluators:
+        used = True
+        for name in e.input_layers:
+            if name not in layer_names:
+                used = False
+                break
+        if used:
+            evaluator_names.add(e.name)
+    return evaluator_names
+
+
+def __trim_submodel__(old_submodel, layer_names, input_layer_names,
+                      output_layer_names, evaluator_names):
+
+    submodel = SubModelConfig()
+    submodel.name = old_submodel.name
+    submodel.layer_names.extend(
+        filter(lambda x: x in layer_names, old_submodel.layer_names))
+    submodel.input_layer_names.extend(
+        filter(lambda x: x in input_layer_names, submodel.layer_names))
+    submodel.output_layer_names.extend(
+        filter(lambda x: x in output_layer_names, submodel.layer_names))
+    submodel.evaluator_names.extend(
+        filter(lambda x: x in evaluator_names, old_submodel.evaluator_names))
+
+    submodel.is_recurrent_layer_group = old_submodel.is_recurrent_layer_group
+    submodel.reversed = old_submodel.reversed
+
+    submodel.memories.extend(
+        filter(lambda x: x.link_name in layer_names, old_submodel.memories))
+    target_inlinkid = (old_submodel.target_inlinkid
+                       if old_submodel.HasField('target_inlinkid') else -1)
+    in_links = []
+    for i, link in enumerate(old_submodel.in_links):
+        if link.link_name in layer_names or i == target_inlinkid:
+            in_links.append(link)
+            if i == target_inlinkid:
+                target_inlinkid = len(in_links) - 1
+    submodel.in_links.extend(in_links)
+
+    submodel.out_links.extend(
+        filter(lambda x: x.link_name in layer_names, old_submodel.out_links))
+    if old_submodel.HasField('generator'):
+        submodel.generator.CopyFrom(old_submodel.generator)
+
+    if old_submodel.HasField('target_inlinkid'):
+        submodel.target_inlinkid = target_inlinkid
+    return submodel
+
+
+def parse_network(output_layers, extra_layers=None):
+    if not isinstance(output_layers, collections.Sequence):
+        output_layers = [output_layers]
+    if extra_layers is not None and not isinstance(extra_layers,
+                                                   collections.Sequence):
+        extra_layers = [extra_layers]
+    else:
+        extra_layers = []
 
-        predict = gipt.after_real_step(step(*args))
+    layer_names = __get_used_layers__(output_layers + extra_layers)
+    submodel_names = __get_used_submodels__(layer_names)
+    submodel_names.add('root')
+    evaluator_names = __get_used_evaluators__(layer_names)
+    input_layer_names = set()
+    output_layer_names = set()
 
-        eos_layer = eos(input=predict, eos_id=eos_id, name=eos_name)
-        predict.append_child(layer=eos_layer, parent_names=[predict.name])
+    model_config = ModelConfig()
+    model_config.type = cp.g_config.model_config.type
+    for l in cp.g_config.model_config.layers:
+        if l.name not in layer_names:
+            continue
+        model_config.layers.extend([l])
+        if l.type == 'data':
+            model_config.input_layer_names.append(l.name)
+            input_layer_names.add(l.name)
 
-        return predict
+    for layer in output_layers:
+        model_config.output_layer_names.append(layer.full_name)
+        output_layer_names.add(layer.full_name)
 
-    # tmp = paddle.layer.recurrent_group(
-    #     step=__real_step__,
-    #     input=real_input,
-    #     reverse=False,
-    #     name=name,
-    #     is_generating=True)
-    tmp = recurrent_group(step=__real_step__, input=real_input, name=name)
+    for e in cp.g_config.model_config.evaluators:
+        if e.name in evaluator_names:
+            model_config.evaluators.extend([e])
 
-    return tmp
+    for s in cp.g_config.model_config.sub_models:
+        if s.name in submodel_names:
+            s = __trim_submodel__(s, layer_names, input_layer_names,
+                                  output_layer_names, evaluator_names)
+            model_config.sub_models.extend([s])
 
+    parameter_names = __get_used_parameters__(layer_names,
+                                              model_config.sub_models)
 
-beam_search.__doc__ = conf_helps.beam_search.__doc__
+    for p in cp.g_config.model_config.parameters:
+        if p.name in parameter_names:
+            model_config.parameters.extend([p])
 
-__projection_names__ = filter(lambda x: x.endswith('_projection'),
-                              dir(conf_helps))
+    return model_config
 
-__all__ += __projection_names__
 
-__operator_names__ = filter(lambda x: x.endswith('_operator'), dir(conf_helps))
-__all__ += __operator_names__
+def get_layer(name):
+    return config_base.__layer_map__.get(name)
 
-# convert projection
-for prj in __projection_names__:
-    globals()[prj] = __convert_to_v2__(
-        prj, parent_names=['input'], is_default_name=False)
-    globals()[prj].__name__ = prj
 
-# convert operator
-operator_list = [
-    # [V1_method_name, parent_names],
-    ['dotmul_operator', ['a', 'b']],
-    ['conv_operator', ['img', 'filter']]
-]
-for op in operator_list:
-    globals()[op[0]] = __convert_to_v2__(
-        op[0], parent_names=op[1], is_default_name=False)
-    globals()[op[0]].__name__ = op[0]
+cp.begin_parse()

python/paddle/v2/networks.py

@@ -24,20 +24,7 @@ def __initialize__():
         if each_subnetwork in ['inputs', 'outputs']:
             continue
         func = getattr(conf_nw, each_subnetwork)
-        if hasattr(func, 'argspec'):
-            argspec = func.argspec
-        else:
-            argspec = inspect.getargspec(func)
-        if each_subnetwork == 'simple_attention':
-            parents = ['encoded_sequence', 'encoded_proj', 'decoder_state']
-        else:
-            parents = filter(lambda x: x.startswith('input'), argspec.args)
-        assert len(parents) != 0, each_subnetwork
-        v2_subnet = __convert_to_v2__(
-            each_subnetwork,
-            parent_names=parents,
-            is_default_name='name' in argspec.args)
-        globals()[each_subnetwork] = v2_subnet
+        globals()[each_subnetwork] = func
         globals()[each_subnetwork].__name__ = each_subnetwork
         global __all__
         __all__.append(each_subnetwork)

python/paddle/v2/op.py

+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import layer
+import activation as act
+from config_base import Layer
+from paddle.trainer_config_helpers.attrs import is_compatible_with
+from paddle.trainer_config_helpers.default_decorators import wrap_name_default
+
+__all__ = []
+
+
+def __register_unary_math_op__(op_name, act):
+    def op(input, name=None):
+        return layer.mixed(
+            input=[layer.identity_projection(input=input)], name=name, act=act)
+
+    op = wrap_name_default(op_name)(op)
+    op.__doc__ = type(act).__doc__
+    globals()[op_name] = op
+    __all__.append(op_name)
+
+
+__register_unary_math_op__('exp', act.Exp())
+__register_unary_math_op__('log', act.Log())
+__register_unary_math_op__('abs', act.Abs())
+__register_unary_math_op__('sigmoid', act.Sigmoid())
+__register_unary_math_op__('tanh', act.Tanh())
+__register_unary_math_op__('square', act.Square())
+__register_unary_math_op__('relu', act.Relu())
+__register_unary_math_op__('sqrt', act.Sqrt())
+__register_unary_math_op__('reciprocal', act.Reciprocal())
+__register_unary_math_op__('softmax', act.Softmax())
+
+
+def __add__(layeroutput, other):
+    if is_compatible_with(other, float):
+        return layer.slope_intercept(input=layeroutput, intercept=other)
+    if not isinstance(other, Layer):
+        raise TypeError("Layer can only be added with"
+                        " another Layer or a number")
+    if layeroutput.size == other.size:
+        return layer.mixed(input=[
+            layer.identity_projection(input=layeroutput),
+            layer.identity_projection(input=other)
+        ])
+    if other.size != 1 and layeroutput.size != 1:
+        raise TypeError("Two Layer can be added only if they have equal size"
+                        " or one of their sizes is 1. sizes are %s and %s" %
+                        (layeroutput.size, other.size))
+    elif layeroutput.size == 1:
+        tmp = layeroutput
+        layeroutput = other
+        other = tmp
+    other = layer.repeat(other, layeroutput.size)
+    return layer.mixed(input=[
+        layer.identity_projection(input=layeroutput),
+        layer.identity_projection(input=other)
+    ])
+
+
+Layer.__radd__ = __add__
+Layer.__add__ = __add__
+
+
+def __neg__(layeroutput):
+    return layer.slope_intercept(input=layeroutput, slope=-1.0)
+
+
+Layer.__neg__ = __neg__
+
+
+def __sub__(layeroutput, other):
+    if is_compatible_with(other, float):
+        return layer.slope_intercept(input=layeroutput, intercept=other)
+    if not isinstance(other, Layer):
+        raise TypeError("Layer can only be subtracted with"
+                        " another Layeroutput or a number")
+    return __add__(layeroutput, -other)
+
+
+Layer.__sub__ = __sub__
+
+
+def __rsub__(layeroutput, other):
+    neg = layer.slope_intercept(input=layeroutput, slope=-1.0)
+    return __add__(neg, other)
+
+
+Layer.__rsub__ = __rsub__
+
+
+def __mul__(layeroutput, other):
+    if is_compatible_with(other, float):
+        return layer.slope_intercept(input=layeroutput, slope=other)
+    if not isinstance(other, Layer):
+        raise TypeError("Layer can only be multiplied with"
+                        " another Layer or a number")
+    elif layeroutput.size == 1:
+        return layer.scaling(input=other, weight=layeroutput)
+    elif other.size == 1:
+        return layer.scaling(input=layeroutput, weight=other)
+    else:
+        raise TypeError("At least one of the operand of '*' must be a number"
+                        " or a Layer with size=1")
+
+
+Layer.__mul__ = __mul__
+Layer.__rmul__ = __mul__

python/paddle/v2/tests/CMakeLists.txt

-add_python_test(test_v2_api test_data_feeder.py test_parameters.py
+add_python_test(test_v2_api test_data_feeder.py test_op.py test_parameters.py
 test_layer.py test_rnn_layer.py test_topology.py test_image.py)

python/paddle/v2/tests/test_layer.py

@@ -73,7 +73,7 @@ class AggregateLayerTest(unittest.TestCase):
         pool = layer.pooling(
             input=pixel,
             pooling_type=pooling.Avg(),
-            agg_level=layer.AggregateLevel.EACH_SEQUENCE)
+            agg_level=layer.AggregateLevel.TO_SEQUENCE)
         last_seq = layer.last_seq(input=pixel)
         first_seq = layer.first_seq(input=pixel)
         concat = layer.concat(input=[last_seq, first_seq])
@@ -109,7 +109,7 @@ class ReshapeLayerTest(unittest.TestCase):
         expand = layer.expand(
             input=weight,
             expand_as=pixel,
-            expand_level=layer.ExpandLevel.FROM_TIMESTEP)
+            expand_level=layer.ExpandLevel.FROM_NO_SEQUENCE)
         repeat = layer.repeat(input=pixel, num_repeats=4)
         reshape = layer.seq_reshape(input=pixel, reshape_size=4)
         rotate = layer.rotate(input=pixel, height=16, width=49)
@@ -173,9 +173,9 @@ class OtherLayerTest(unittest.TestCase):
 
 class ProjOpTest(unittest.TestCase):
     def test_projection(self):
-        input = layer.data(name='data', type=data_type.dense_vector(784))
+        input = layer.data(name='data2', type=data_type.dense_vector(784))
         word = layer.data(
-            name='word', type=data_type.integer_value_sequence(10000))
+            name='word2', type=data_type.integer_value_sequence(10000))
         fc0 = layer.fc(input=input, size=100, act=activation.Sigmoid())
         fc1 = layer.fc(input=input, size=200, act=activation.Sigmoid())
         mixed0 = layer.mixed(
@@ -204,8 +204,8 @@ class ProjOpTest(unittest.TestCase):
             dotmul1 += dotmul
 
         context = layer.context_projection(input=fc0, context_len=5)
-        context0 = layer.mixed(size=100, input=context)
-        with layer.mixed(size=100) as context1:
+        context0 = layer.mixed(size=500, input=context)
+        with layer.mixed(size=500) as context1:
             context1 += context
 
         conv = layer.conv_projection(
@@ -231,8 +231,8 @@ class ProjOpTest(unittest.TestCase):
         print layer.parse_network(conv1)
 
     def test_operator(self):
-        ipt0 = layer.data(name='data', type=data_type.dense_vector(784))
-        ipt1 = layer.data(name='word', type=data_type.dense_vector(128))
+        ipt0 = layer.data(name='data1', type=data_type.dense_vector(784))
+        ipt1 = layer.data(name='word1', type=data_type.dense_vector(128))
         fc0 = layer.fc(input=ipt0, size=100, act=activation.Sigmoid())
         fc1 = layer.fc(input=ipt0, size=100, act=activation.Sigmoid())
 
@@ -261,7 +261,7 @@ class ProjOpTest(unittest.TestCase):
 
 class NetworkTests(unittest.TestCase):
     def test_vgg(self):
-        img = layer.data(name='pixel', type=data_type.dense_vector(784))
+        img = layer.data(name='pixel1', type=data_type.dense_vector(784))
         vgg_out = networks.small_vgg(
             input_image=img, num_channels=1, num_classes=2)
         print layer.parse_network(vgg_out)
@@ -269,12 +269,12 @@ class NetworkTests(unittest.TestCase):
 
 class EvaluatorTest(unittest.TestCase):
     def test_evaluator(self):
-        img = layer.data(name='pixel', type=data_type.dense_vector(784))
+        img = layer.data(name='pixel2', type=data_type.dense_vector(784))
         output = layer.fc(input=img,
                           size=10,
                           act=activation.Softmax(),
                           name='fc_here')
-        lbl = layer.data(name='label', type=data_type.integer_value(10))
+        lbl = layer.data(name='label2', type=data_type.integer_value(10))
         cost = layer.cross_entropy_cost(input=output, label=lbl)
 
         evaluator.classification_error(input=output, label=lbl)

python/paddle/trainer_config_helpers/config_parser.py → python/paddle/v2/tests/test_op.py

-# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+# Copyright PaddlePaddle contributors. All Rights Reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,28 +11,40 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import paddle.trainer.config_parser as config_parser
-'''
-This file is a wrapper of formal config_parser. The main idea of this file is to 
-separete different config logic into different function, such as network configuration
- and optimizer configuration.
-'''
-
-__all__ = [
-    "parse_trainer_config", "parse_network_config", "parse_optimizer_config"
-]
-
-
-def parse_trainer_config(trainer_conf, config_arg_str):
-    return config_parser.parse_config(trainer_conf, config_arg_str)
-
-
-def parse_network_config(network_conf):
-    config = config_parser.parse_config(network_conf, '')
-    return config.model_config
-
-
-def parse_optimizer_config(optimizer_conf):
-    config = config_parser.parse_config(optimizer_conf, '')
-    return config.opt_config
+import unittest
+
+import paddle.v2.data_type as data_type
+import paddle.v2.layer as layer
+import paddle.v2.op as op
+
+
+class OpTest(unittest.TestCase):
+    def test_op(self):
+        x = layer.data(name='data', type=data_type.dense_vector(128))
+        x = op.exp(x)
+        x = op.sqrt(x)
+        x = op.reciprocal(x)
+        x = op.log(x)
+        x = op.abs(x)
+        x = op.sigmoid(x)
+        x = op.tanh(x)
+        x = op.square(x)
+        x = op.relu(x)
+        y = 1 + x
+        y = y + 1
+        y = x + y
+        y = y - x
+        y = y - 2
+        y = 2 - y
+        y = 2 * y
+        y = y * 3
+        z = layer.data(name='data_2', type=data_type.dense_vector(1))
+        y = y * z
+        y = z * y
+        y = y + z
+        y = z + y
+        print layer.parse_network(y)
+
+
+if __name__ == '__main__':
+    unittest.main()

python/paddle/v2/tests/test_rnn_layer.py

@@ -20,6 +20,8 @@ import paddle.v2.data_type as data_type
 import paddle.v2.layer as layer
 from paddle.trainer_config_helpers.config_parser_utils import \
     parse_network_config as parse_network
+from paddle.trainer_config_helpers.config_parser_utils import \
+    reset_parser
 
 
 class RNNTest(unittest.TestCase):
@@ -29,6 +31,8 @@ class RNNTest(unittest.TestCase):
         hidden_dim = 8
 
         def parse_old_rnn():
+            reset_parser()
+
             def step(y):
                 mem = conf_helps.memory(name="rnn_state", size=hidden_dim)
                 out = conf_helps.fc_layer(
@@ -48,6 +52,8 @@ class RNNTest(unittest.TestCase):
             return str(parse_network(test))
 
         def parse_new_rnn():
+            reset_parser()
+
             def new_step(y):
                 mem = layer.memory(name="rnn_state", size=hidden_dim)
                 out = layer.fc(input=[y, mem],
@@ -75,6 +81,8 @@ class RNNTest(unittest.TestCase):
         label_dim = 3
 
         def parse_old_rnn():
+            reset_parser()
+
             def test():
                 data = conf_helps.data_layer(name="word", size=dict_dim)
                 label = conf_helps.data_layer(name="label", size=label_dim)
@@ -114,6 +122,7 @@ class RNNTest(unittest.TestCase):
             return str(parse_network(test))
 
         def parse_new_rnn():
+            reset_parser()
             data = layer.data(
                 name="word", type=data_type.dense_vector(dict_dim))
             label = layer.data(

python/paddle/v2/tests/test_topology.py

@@ -46,8 +46,8 @@ class TestTopology(unittest.TestCase):
         self.assertEqual(label_data_type[1].dim, 10)
 
     def test_get_layer(self):
-        pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
-        label = layer.data(name='label', type=data_type.integer_value(10))
+        pixel = layer.data(name='pixel2', type=data_type.dense_vector(784))
+        label = layer.data(name='label2', type=data_type.integer_value(10))
         hidden = layer.fc(input=pixel,
                           size=100,
                           act=conf_helps.SigmoidActivation())
@@ -56,14 +56,14 @@ class TestTopology(unittest.TestCase):
                              act=conf_helps.SoftmaxActivation())
         cost = layer.classification_cost(input=inference, label=label)
         topo = topology.Topology(cost)
-        pixel_layer = topo.get_layer("pixel")
-        label_layer = topo.get_layer("label")
+        pixel_layer = topo.get_layer("pixel2")
+        label_layer = topo.get_layer("label2")
         self.assertEqual(pixel_layer, pixel)
         self.assertEqual(label_layer, label)
 
     def test_parse(self):
-        pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
-        label = layer.data(name='label', type=data_type.integer_value(10))
+        pixel = layer.data(name='pixel3', type=data_type.dense_vector(784))
+        label = layer.data(name='label3', type=data_type.integer_value(10))
         hidden = layer.fc(input=pixel,
                           size=100,
                           act=conf_helps.SigmoidActivation())

python/paddle/v2/topology.py

@@ -15,36 +15,13 @@
 import collections
 
 from paddle.proto.ModelConfig_pb2 import ModelConfig
-
+import paddle.trainer_config_helpers as conf_helps
 import layer as v2_layer
+import config_base
 
 __all__ = ['Topology']
 
 
-def __flatten__(lis):
-    """
-    Given a list, possibly nested to any level, return it flattened.
-    """
-    new_lis = []
-    for item in lis:
-        if isinstance(item, collections.Sequence):
-            new_lis.extend(__flatten__(item))
-        else:
-            new_lis.append(item)
-    return new_lis
-
-
-def __bfs_travel__(callback, *layers):
-    layers = __flatten__(layers)
-    for each_layer in layers:
-        __break__ = callback(each_layer)
-        if __break__:
-            return
-        __layers__ = each_layer.__parent_layers__.values() + \
-                     each_layer.extra_parent()
-        __bfs_travel__(callback, *__layers__)
-
-
 class Topology(object):
     """
     Topology is used to store the information about all layers
@@ -94,31 +71,18 @@ class Topology(object):
         :param name:
         :return:
         """
-        result_layer = [None]
-
-        def __impl__(l):
-            if l.name == name:
-                result_layer[0] = l
-                return True  # break
-            return False
-
-        __bfs_travel__(__impl__, *self.layers)
-        if result_layer[0] is None:
-            raise ValueError("No such layer %s" % name)
-        return result_layer[0]
+        return v2_layer.get_layer(name)
 
     def data_layers(self):
         """
         get all data layer
         :return:
         """
-        data_layers = dict()
-
-        def __impl__(l):
-            if isinstance(l, v2_layer.DataLayerV2):
-                data_layers[l.name] = l
-
-        __bfs_travel__(__impl__, *self.layers)
+        data_layers = {}
+        for layer in self.proto().layers:
+            l = v2_layer.get_layer(layer.name)
+            if l and l.layer_type == conf_helps.LayerType.DATA:
+                data_layers[layer.name] = l
         return data_layers
 
     def data_type(self):
@@ -127,7 +91,7 @@ class Topology(object):
         [('image', dense_vector(768)), ('label', integer_value(10))]
         """
         data_layers = self.data_layers()
-        return [(nm, data_layers[nm].type)
+        return [(nm, data_layers[nm].data_type)
                 for nm in self.proto().input_layer_names]
 
     def get_layer_proto(self, name):
@@ -138,5 +102,5 @@ class Topology(object):
 
 
 def __check_layer_type__(layer):
-    if not isinstance(layer, v2_layer.LayerV2):
-        raise ValueError('layer should have type paddle.layer.Layer')
+    if not isinstance(layer, config_base.Layer):
+        raise ValueError('layer should have type paddle.v2.config_base.Layer')