Merge branch 'develop' into go_optimizer

c3833131 · dzhwinter · GitHub · 36f8b848 · 3f445782 · c3833131
10 changed file
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -130,14 +130,16 @@ if(WITH_GPU)
 endif(WITH_GPU)

 add_subdirectory(proto)
-add_subdirectory(paddle)
-add_subdirectory(python)

+# "add_subdirectory(paddle)" and "add_subdirectory(python)" should be
+# placed after this block, because they depends on it.
 if(WITH_GOLANG)
-    #TODO (add go/master/c back when fixed)
-    add_subdirectory(go/pserver/cclient)
+  add_subdirectory(go/master/c)
+  add_subdirectory(go/pserver/cclient)
 endif(WITH_GOLANG)

+add_subdirectory(paddle)
+add_subdirectory(python)
 if(WITH_DOC)
    add_subdirectory(doc)
 endif()
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -69,3 +69,27 @@ endif(NOT WITH_GPU)

 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SIMD_FLAG}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SIMD_FLAG}")
+
+if(WITH_GOLANG)
+  # we need to symlink Paddle directory into GOPATH. If we
+  # don't do it and we have code that depends on Paddle, go
+  # get ./... will download a new Paddle repo from Github,
+  # without the changes in our current Paddle repo that we
+  # want to build.
+  set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
+  file(MAKE_DIRECTORY ${GOPATH})
+  set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle")
+  add_custom_target(go_path)
+  add_custom_command(TARGET go_path
+    # Symlink Paddle directory into GOPATH
+    COMMAND mkdir -p ${PADDLE_IN_GOPATH}
+    COMMAND rm -rf ${PADDLE_IN_GOPATH}
+    COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
+    # Automatically get all dependencies specified in the source code
+    # We can't run `go get -d ./...` for every target, because
+    # multiple `go get` can not run concurrently, but make need to be
+    # able to run with multiple jobs.
+    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./go/...
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  )
+endif(WITH_GOLANG)
--- a/cmake/external/python.cmake
+++ b/cmake/external/python.cmake
@@ -32,193 +32,6 @@ IF(PYTHONINTERP_FOUND)
        MESSAGE(FATAL_ERROR "Found Python Protobuf ${PY_GOOGLE.PROTOBUF_VERSION} < 3.0.0, "
        "please use pip to upgrade protobuf. pip install -U protobuf")
    ENDIF()
-ELSE(PYTHONINTERP_FOUND)
-    MESSAGE(FATAL_ERROR "Please install python 2.7 before building PaddlePaddle.")
-    ##################################### PYTHON ########################################
-    SET(PYTHON_SOURCES_DIR ${THIRD_PARTY_PATH}/python)
-    SET(PYTHON_INSTALL_DIR ${THIRD_PARTY_PATH}/install/python)
-    SET(_python_DIR ${PYTHON_INSTALL_DIR})
-
-    IF(UNIX)
-        SET(PYTHON_FOUND ON)
-        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE)
-        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE)
-        SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE)
-        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE)
-    ELSEIF(WIN32)
-        SET(PYTHON_FOUND ON)
-        SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE)
-        SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE)
-        SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE)
-        SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE)
-    ELSE()
-        MESSAGE(FATAL_ERROR "Unknown system !")
-    ENDIF()
-
-    IF(APPLE)
-        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
-            -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
-            )
-    ENDIF()
-
-    SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS)
-
-    # Force Python build to "Release".
-    IF(CMAKE_CONFIGURATION_TYPES)
-        SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR})
-        SET(CMAKE_CFG_INTDIR "Release")
-    ELSE()
-        LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS
-            -DCMAKE_BUILD_TYPE:STRING=Release
-            )
-    ENDIF()
-
-    ExternalProject_Add(python
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        GIT_REPOSITORY    "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git"
-        PREFIX            ${PYTHON_SOURCES_DIR}
-        UPDATE_COMMAND    ""
-        CMAKE_ARGS        -DPYTHON_VERSION=2.7.12
-        CMAKE_ARGS        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-        CMAKE_ARGS        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-        CMAKE_CACHE_ARGS
-            -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR}
-            -DBUILD_LIBPYTHON_SHARED:BOOL=OFF
-            -DUSE_SYSTEM_LIBRARIES:BOOL=OFF
-            -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT}
-            -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR}
-            -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES}
-            -DDOWNLOAD_SOURCES:BOOL=ON
-            -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF
-            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS}
-            ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
-        DEPENDS zlib
-    )
-
-    SET(py_env
-        PATH=${PYTHON_INSTALL_DIR}/bin
-        PYTHONHOME=${PYTHON_INSTALL_DIR}
-        PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
-    ####################################################################################
-
-    ##################################### SETUPTOOLS ###################################
-    SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools)
-    ExternalProject_Add(setuptools
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        PREFIX              ${SETUPTOOLS_SOURCES_DIR}
-        URL                 "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz"
-        BUILD_IN_SOURCE     1
-        PATCH_COMMAND       ""
-        UPDATE_COMMAND      ""
-        CONFIGURE_COMMAND   ""
-        INSTALL_COMMAND     ""
-        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        DEPENDS             python zlib
-    )
-    #####################################################################################
-
-    ##################################### SIX ###########################################
-    SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six)
-    ExternalProject_Add(six
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        PREFIX              ${SIX_SOURCES_DIR}
-        URL                 https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz
-        BUILD_IN_SOURCE     1
-        PATCH_COMMAND       ""
-        UPDATE_COMMAND      ""
-        CONFIGURE_COMMAND   ""
-        INSTALL_COMMAND     ""
-        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        DEPENDS             python setuptools
-    )
-    #####################################################################################
-
-    ##################################### CYTHON ########################################
-    SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython)
-    ExternalProject_Add(cython
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        PREFIX                ${CYTHON_SOURCES_DIR}
-        URL                   https://github.com/cython/cython/archive/0.25.2.tar.gz
-        GIT_TAG               0.25.2
-        BUILD_IN_SOURCE       1
-        CONFIGURE_COMMAND     ""
-        PATCH_COMMAND         ""
-        UPDATE_COMMAND        ""
-        INSTALL_COMMAND       ""
-        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        DEPENDS               python
-    )
-    ####################################################################################
-
-    ##################################### NUMPY ########################################
-    SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy)
-    SET(NUMPY_TAG_VERSION "v1.11.3")
-    SET(NUMPY_VERSION "1.11.3")
-
-    SET(EGG_NAME "")
-    SET(PYTHON_NUMPY_INCLUDE_DIR "")
-    IF(WIN32)
-        SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg")
-    ELSE(WIN32)
-        IF(APPLE)
-            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}")
-        ELSE(APPLE)
-            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
-            SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux")
-        ENDIF(APPLE)
-
-        FOREACH(suffix x86_64 intel fat64 fat32 universal)
-            LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include)
-        ENDFOREACH()
-    ENDIF(WIN32)
-
-    ExternalProject_Add(numpy
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        GIT_REPOSITORY      https://github.com/numpy/numpy.git
-        GIT_TAG             ${NUMPY_TAG_VERSION}
-        CONFIGURE_COMMAND   ""
-        UPDATE_COMMAND      ""
-        PREFIX              ${NUMPY_SOURCES_DIR}
-        BUILD_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        BUILD_IN_SOURCE     1
-        DEPENDS             python setuptools cython
-    )
-    ####################################################################################
-
-    ##################################### WHEEL ########################################
-    SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel)
-    ExternalProject_Add(wheel
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        URL                 https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz
-        PREFIX              ${WHEEL_SOURCES_DIR}
-        CONFIGURE_COMMAND   ""
-        UPDATE_COMMAND      ""
-        BUILD_COMMAND       ""
-        INSTALL_COMMAND     env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        BUILD_IN_SOURCE     1
-        DEPENDS             python setuptools
-    )
-    ####################################################################################
-
-    ################################### PROTOBUF #######################################
-    SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf)
-    ExternalProject_Add(python-protobuf
-        ${EXTERNAL_PROJECT_LOG_ARGS}
-        URL                   https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz
-        URL_MD5               38b5fb160c768d2f8444d0c6d637ff91
-        PREFIX                ${PY_PROTOBUF_SOURCES_DIR}
-        BUILD_IN_SOURCE       1
-        PATCH_COMMAND         ""
-        CONFIGURE_COMMAND     ""
-        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-        INSTALL_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
-        DEPENDS               python setuptools six
-    )
-    ####################################################################################
-
-    LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
-
 ENDIF(PYTHONINTERP_FOUND)

 IF(WITH_PYTHON)

--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -146,9 +146,9 @@ function(merge_static_libs TARGET_NAME)
    set(outlibfile "$<TARGET_FILE:${TARGET_NAME}>")

    foreach(lib ${libs})
-    add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
-      COMMAND ${CMAKE_AR} ru ${outlibfile} @"../${objlistfile}"
-      WORKING_DIRECTORY ${objdir})
+      add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
+      COMMAND ${CMAKE_AR} ru ${outlibfile} @"../${lib}.objlist"
+      WORKING_DIRECTORY ${lib}.objdir)
    endforeach()

    add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
@@ -256,10 +256,6 @@ function(nv_test TARGET_NAME)
  endif()
 endfunction(nv_test)

-set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
-file(MAKE_DIRECTORY ${GOPATH})
-set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle")
-
 function(go_library TARGET_NAME)
  set(options STATIC static SHARED shared)
  set(oneValueArgs "")
@@ -268,10 +264,10 @@ function(go_library TARGET_NAME)

  if (go_library_SHARED OR go_library_shared)
    set(BUILD_MODE "-buildmode=c-shared")
-    set(LIB_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(${TARGET_NAME}_LIB_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" CACHE STRING "output library name for target ${TARGET_NAME}")
  else()
    set(BUILD_MODE "-buildmode=c-archive")
-    set(LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(${TARGET_NAME}_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${TARGET_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE STRING "output library name for target ${TARGET_NAME}")
  endif()

  # Add dummy code to support `make target_name` under Terminal Command
@@ -286,25 +282,17 @@ function(go_library TARGET_NAME)
    add_dependencies(${TARGET_NAME} ${go_library_DEPS})
  endif(go_library_DEPS)

-  # we need to symlink Paddle directory into GOPATH. If we
-  # don't do it and we have code that depends on Paddle, go
-  # get ./... will download a new Paddle repo from Github,
-  # without the changes in our current Paddle repo that we
-  # want to build.
+  set(${TARGET_NAME}_LIB_PATH "${CMAKE_CURRENT_BINARY_DIR}/${${TARGET_NAME}_LIB_NAME}" CACHE STRING "output library path for target ${TARGET_NAME}")
+
  file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
  add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
-    COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
-    # Symlink Paddle directory into GOPATH
-    COMMAND mkdir -p ${PADDLE_IN_GOPATH}
-    COMMAND rm -rf ${PADDLE_IN_GOPATH}                                                                                                                                         
-    COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
-    # Automatically get all dependencies specified in the source code                                                                                                                                 
-    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./...
+    COMMAND rm "${${TARGET_NAME}_LIB_PATH}"
    # Golang build source code
    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
-    -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
+    -o "${${TARGET_NAME}_LIB_PATH}"
    ${GO_SOURCE}
    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  add_dependencies(${TARGET_NAME} go_path)
 endfunction(go_library)

 function(go_binary TARGET_NAME)

--- a/go/master/c/CMakeLists.txt
+++ b/go/master/c/CMakeLists.txt
 cmake_minimum_required(VERSION 3.0)

-get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY)
-get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY)
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake")
-
-project(cxx_go C Go)
-
-include(golang)
-include(flags)
-
-set(MASTER_LIB_NAME "paddle_master")
-go_library(${MASTER_LIB_NAME} SHARED)
-
-if(PROJ_ROOT)
-  add_custom_command(OUTPUT ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so
-    COMMAND rm ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.h
-    COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.so ${PROJ_ROOT}/python/paddle/v2/master/
-    DEPENDS ${MASTER_LIB_NAME})
-  add_custom_target(paddle_master_shared ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so)
-endif(PROJ_ROOT)
+go_library(paddle_master SHARED)
--- a/go/pserver/cclient/test/CMakeLists.txt
+++ b/go/pserver/cclient/test/CMakeLists.txt
-cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient)
\ No newline at end of file
+cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient)
+
--- a/paddle/trainer/CMakeLists.txt
+++ b/paddle/trainer/CMakeLists.txt
@@ -73,6 +73,6 @@ endif()

 if(WITH_GOLANG)
  add_dependencies(paddle_trainer_lib paddle_pserver_cclient)
-  target_link_libraries(paddle_trainer ${CMAKE_BINARY_DIR}/go/pserver/cclient/libpaddle_pserver_cclient.a)
-  target_link_libraries(paddle_trainer_lib ${CMAKE_BINARY_DIR}/go/pserver/cclient/libpaddle_pserver_cclient.a)
+  target_link_libraries(paddle_trainer paddle_pserver_cclient)
+  target_link_libraries(paddle_trainer_lib paddle_pserver_cclient)
 endif(WITH_GOLANG)
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -7,10 +7,18 @@ file(GLOB UTILS_PY_FILES . ./paddle/utils/*.py)
 file(GLOB_RECURSE V2_PY_FILES ./paddle/v2/ *.py)

 set(PY_FILES paddle/__init__.py
-             ${TRAINER_PY_FILES}
-             ${HELPERS_PY_FILES}
-             ${UTILS_PY_FILES}
-             ${V2_PY_FILES})
+  ${TRAINER_PY_FILES}
+  ${HELPERS_PY_FILES}
+  ${UTILS_PY_FILES}
+  ${V2_PY_FILES})
+
+add_custom_target(copy_paddle_master)
+if(WITH_GOLANG)
+  add_custom_command(TARGET copy_paddle_master
+    COMMAND cp ${paddle_master_LIB_PATH} ${PROJ_ROOT}/python/paddle/v2/master/
+    )
+  add_dependencies(copy_paddle_master paddle_master)
+endif(WITH_GOLANG)

 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
    ${CMAKE_CURRENT_BINARY_DIR}/setup.py)
@@ -18,7 +26,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
 add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
    COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
    COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp
-    DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies})
+    DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies} copy_paddle_master)

 add_custom_target(paddle_python ALL DEPENDS
    ${OUTPUT_DIR}/.timestamp)

--- a/python/paddle/v2/__init__.py
+++ b/python/paddle/v2/__init__.py
@@ -56,6 +56,7 @@ __all__ = [
    'plot',
    'evaluator',
    'image',
+    'master',
 ]



--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -8,6 +8,7 @@ packages=['paddle',
          'paddle.v2',
          'paddle.v2.dataset',
          'paddle.v2.reader',
+          'paddle.v2.master',
          'paddle.v2.plot']

 setup_requires=["requests",
@@ -26,6 +27,7 @@ setup(name='paddle',
      description='Parallel Distributed Deep Learning',
      install_requires=setup_requires,
      packages=packages,
+      package_data={'paddle.v2.master': ['${paddle_master_LIB_NAME}'], },
      package_dir={
          '': '${CMAKE_CURRENT_SOURCE_DIR}'
      },