Merge pull request #2530 from jacquesqiao/fix-go

Add WITH_GOLANG to control the link of go lib

CMakeLists.txt

@@ -47,6 +47,7 @@ option(WITH_COVERAGE    "Compile PaddlePaddle with code coverage"       OFF)
 option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
 option(ON_TRAVIS        "Exclude special unit test on Travis CI"        OFF)
 option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
+option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
 
 # CMAKE_BUILD_TYPE
 if(NOT CMAKE_BUILD_TYPE)
@@ -107,6 +108,7 @@ include(configure)          # add paddle env configuration
 include_directories("${PROJ_ROOT}")
 include_directories("${PROJ_ROOT}/paddle/cuda/include")
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto")
+include_directories("${CMAKE_CURRENT_BINARY_DIR}/go/pserver/cclient")
 
 set(EXTERNAL_LIBS
     ${GFLAGS_LIBRARIES}
@@ -126,9 +128,12 @@ endif(WITH_GPU)
 
 add_subdirectory(proto)
 add_subdirectory(paddle)
-add_subdirectory(go/master/c)
 add_subdirectory(python)
-add_subdirectory(go/pserver/cclient)
+
+if(WITH_GOLANG)
+    #TODO (add go/master/c back when fixed)
+    add_subdirectory(go/pserver/cclient)
+endif(WITH_GOLANG)
 
 if(WITH_DOC)
     add_subdirectory(doc)

cmake/configure.cmake

@@ -40,6 +40,10 @@ if(NOT CMAKE_CROSSCOMPILING)
     endif()
 endif()
 
+if(NOT WITH_GOLANG)
+    add_definitions(-DPADDLE_WITHOUT_GOLANG)
+endif(NOT WITH_GOLANG)
+
 if(NOT WITH_GPU)
     add_definitions(-DPADDLE_ONLY_CPU)
     add_definitions(-DHPPL_STUB_FUNC)

go/pserver/cclient/CMakeLists.txt

@@ -11,13 +11,4 @@ include(flags)
 
 go_library(paddle_pserver_cclient STATIC)
 
-if(PROJ_ROOT)
-  add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/trainer/libpaddle_pserver_cclient.a
-          COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/libpaddle_pserver_cclient.h ${PROJ_ROOT}/paddle/trainer/
-          COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/libpaddle_pserver_cclient.a ${PROJ_ROOT}/paddle/trainer/
-          WORKING_DIRECTORY ${PROJ_ROOT}/paddle
-          DEPENDS paddle_pserver_cclient)
-  add_custom_target(paddle_pserver_cclient_lib ALL DEPENDS ${PROJ_ROOT}/paddle/trainer/libpaddle_pserver_cclient.a)
-endif(PROJ_ROOT)
-
 add_subdirectory(test)

paddle/api/CMakeLists.txt

@@ -16,7 +16,7 @@ set(API_HEADER
     Internal.h)
 
 add_library(paddle_api STATIC ${API_SOURCES})
-add_dependencies(paddle_api gen_proto_cpp paddle_pserver_cclient_lib)
+add_dependencies(paddle_api gen_proto_cpp paddle_trainer_lib)
 
 INCLUDE(${SWIG_USE_FILE})
 INCLUDE_DIRECTORIES(${PROJ_ROOT}/paddle)

paddle/api/PaddleAPI.h

@@ -842,7 +842,8 @@ public:
                                                int passCount,
                                                bool useSparseUpdater);
   static ParameterUpdater* createNewRemoteUpdater(
-      OptimizationConfig* config, const std::string pserverSpec);
+      OptimizationConfig* config,
+      const std::string pserverSpec) throw(UnsupportError);
   ~ParameterUpdater();
 
   /**

paddle/api/ParameterUpdater.cpp

@@ -15,7 +15,9 @@ limitations under the License. */
 #include "PaddleAPI.h"
 
 #include "PaddleAPIPrivate.h"
+#ifndef PADDLE_WITHOUT_GOLANG
 #include "paddle/trainer/NewRemoteParameterUpdater.h"
+#endif
 #include "paddle/trainer/RemoteParameterUpdater.h"
 #include "paddle/trainer/ThreadParameterUpdater.h"
 
@@ -30,11 +32,16 @@ ParameterUpdater *ParameterUpdater::createLocalUpdater(
 }
 
 ParameterUpdater *ParameterUpdater::createNewRemoteUpdater(
-    OptimizationConfig *config, const std::string pserverSpec) {
+    OptimizationConfig *config,
+    const std::string pserverSpec) throw(UnsupportError) {
+#ifndef PADDLE_WITHOUT_GOLANG
   auto updater = new ParameterUpdater();
   updater->m->updater.reset(new paddle::NewRemoteParameterUpdater(
       config->m->getConfig(), pserverSpec));
   return updater;
+#else
+  throw UnsupportError();
+#endif
 }
 
 ParameterUpdater *ParameterUpdater::createRemoteUpdater(

paddle/optimizer/CMakeLists.txt

@@ -12,5 +12,7 @@ set(OPITMIZER_SRCS
 add_library(paddle_optimizer STATIC ${OPITMIZER_SRCS})
 add_dependencies(paddle_optimizer gen_proto_cpp)
 
-add_simple_unittest(serialization_test)
-add_simple_unittest(parameter_optimizer_test)
+if(WITH_TESTING)
+  add_simple_unittest(serialization_test)
+  add_simple_unittest(parameter_optimizer_test)
+endif()

paddle/trainer/CMakeLists.txt

@@ -26,6 +26,13 @@ set(TRAINER_HEADERS
         ThreadParameterUpdater.h
         TrainerConfigHelper.h)
 
+if(NOT WITH_GOLANG)
+  list(REMOVE_ITEM TRAINER_SOURCES
+          NewRemoteParameterUpdater.cpp)
+  list(REMOVE_ITEM TRAINER_HEADERS
+          NewRemoteParameterUpdater.h)
+endif()
+
 add_library(paddle_trainer_lib STATIC
     ${TRAINER_SOURCES})
 
@@ -34,7 +41,7 @@ add_style_check_target(paddle_trainer_lib
 add_style_check_target(paddle_trainer_lib
     ${TRAINER_HEADERS})
 add_dependencies(paddle_trainer_lib
-    gen_proto_cpp paddle_pserver_cclient_lib)
+    gen_proto_cpp)
 
 macro(add_paddle_exe TARGET_NAME)
   add_executable(${TARGET_NAME} ${ARGN})
@@ -63,5 +70,8 @@ if(APPLE)
   set(CMAKE_EXE_LINKER_FLAGS "-framework CoreFoundation -framework Security")
 endif()
 
-target_link_libraries(paddle_trainer ${CMAKE_CURRENT_SOURCE_DIR}/libpaddle_pserver_cclient.a)
-target_link_libraries(paddle_trainer_lib ${CMAKE_CURRENT_SOURCE_DIR}/libpaddle_pserver_cclient.a)
+if(WITH_GOLANG)
+  add_dependencies(paddle_trainer_lib paddle_pserver_cclient)
+  target_link_libraries(paddle_trainer ${CMAKE_BINARY_DIR}/go/pserver/cclient/libpaddle_pserver_cclient.a)
+  target_link_libraries(paddle_trainer_lib ${CMAKE_BINARY_DIR}/go/pserver/cclient/libpaddle_pserver_cclient.a)
+endif(WITH_GOLANG)

proto/CMakeLists.txt

@@ -36,10 +36,8 @@ foreach(filename ${proto_filenames})
         DEPENDS ${filename} ${external_project_dependencies})
 endforeach()
 
-include_directories(${CMAKE_CURRENT_BINARY_DIR}/proto)
-
 add_custom_target(gen_proto_cpp ALL DEPENDS ${PROTO_GEN})
 add_custom_target(gen_proto_py ALL DEPENDS ${PROTO_GEN_PY})
-add_library(paddle_proto STATIC
-    ${PROTO_GEN})
+
+add_library(paddle_proto STATIC ${PROTO_GEN})
 target_include_directories(paddle_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

python/CMakeLists.txt

@@ -18,7 +18,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in
 add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp
     COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
     COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp
-    DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies} paddle_master_shared)
+    DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies})
 
 add_custom_target(paddle_python ALL DEPENDS
     ${OUTPUT_DIR}/.timestamp)

python/paddle/v2/__init__.py

@@ -26,7 +26,6 @@ import evaluator
 from . import dataset
 from . import reader
 from . import plot
-from . import master
 import attr
 import op
 import pooling
@@ -57,7 +56,6 @@ __all__ = [
     'plot',
     'evaluator',
     'image',
-    'master',
 ]
 
 

python/setup.py.in

@@ -8,8 +8,7 @@ packages=['paddle',
           'paddle.v2',
           'paddle.v2.dataset',
           'paddle.v2.reader',
-          'paddle.v2.plot',
-          'paddle.v2.master']
+          'paddle.v2.plot']
 
 setup_requires=["requests",
                 "numpy",
@@ -25,7 +24,6 @@ setup(name='paddle',
       description='Parallel Distributed Deep Learning',
       install_requires=setup_requires,
       packages=packages,
-      package_data={'paddle.v2.master': ['libpaddle_master.so'], },
       package_dir={
           '': '${CMAKE_CURRENT_SOURCE_DIR}'
       },