From 8742441fc0f3388a5fde3fb15c810718424e33ff Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Fri, 16 Jun 2017 19:32:24 +0000 Subject: [PATCH] integrate master Python lib with cmake --- CMakeLists.txt | 1 + go/cmake/golang.cmake | 12 ++++------ go/master/c/CMakeLists.txt | 21 ++++++++++++++++ go/master/python/.gitignore | 1 - go/master/python/build.sh | 4 ---- go/master/python/setup.py | 19 --------------- python/CMakeLists.txt | 4 ++-- python/paddle/v2/__init__.py | 24 ++++++++++++++++--- .../paddle/v2/master}/.gitignore | 1 + .../paddle/v2/master}/__init__.py | 0 .../paddle/v2/master}/client.py | 2 +- python/setup.py.in | 7 +++--- 12 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 go/master/c/CMakeLists.txt delete mode 100644 go/master/python/.gitignore delete mode 100755 go/master/python/build.sh delete mode 100644 go/master/python/setup.py rename {go/master/python/paddle_master => python/paddle/v2/master}/.gitignore (64%) rename {go/master/python/paddle_master => python/paddle/v2/master}/__init__.py (100%) rename {go/master/python/paddle_master => python/paddle/v2/master}/client.py (93%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2218be5e..2b6a80ca4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,6 +126,7 @@ endif(WITH_GPU) add_subdirectory(proto) add_subdirectory(paddle) +add_subdirectory(go/master/c) add_subdirectory(python) add_subdirectory(go/pserver/cclient) diff --git a/go/cmake/golang.cmake b/go/cmake/golang.cmake index 7c85fb629..a5a43886f 100644 --- a/go/cmake/golang.cmake +++ b/go/cmake/golang.cmake @@ -26,27 +26,23 @@ function(GO_LIBRARY NAME BUILD_TYPE) # automatically get all dependencies specified in the source code # for given target. - add_custom_target(goGet env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ${rel}/...) + add_custom_target(${NAME}_goGet env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ${rel}/...) # make a symlink that references Paddle inside $GOPATH, so go get # will use the local changes in Paddle rather than checkout Paddle # in github. - add_custom_target(copyPaddle + add_custom_target(${NAME}_copyPaddle COMMAND rm -rf ${PADDLE_IN_GOPATH}/Paddle COMMAND ln -sf ${PADDLE_DIR} ${PADDLE_IN_GOPATH}/Paddle) - add_dependencies(goGet copyPaddle) + add_dependencies(${NAME}_goGet ${NAME}_copyPaddle) add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE} - -gcflags=-shared -asmflags=-shared -installsuffix=_shared -a -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}" ${CMAKE_GO_FLAGS} ${GO_SOURCE} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) add_custom_target(${NAME} ALL DEPENDS ${OUTPUT_DIR}/.timestamp ${ARGN}) - add_dependencies(${NAME} goGet) + add_dependencies(${NAME} ${NAME}_goGet) - if(NOT BUILD_TYPE STREQUAL "STATIC") - install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} DESTINATION bin) - endif() endfunction(GO_LIBRARY) diff --git a/go/master/c/CMakeLists.txt b/go/master/c/CMakeLists.txt new file mode 100644 index 000000000..acce69805 --- /dev/null +++ b/go/master/c/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.0) + +get_filename_component(PARENT_DIR ${CMAKE_CURRENT_SOURCE_DIR} DIRECTORY) +get_filename_component(PARENT_DIR ${PARENT_DIR} DIRECTORY) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PARENT_DIR}/cmake") + +project(cxx_go C Go) + +include(golang) +include(flags) + +set(MASTER_LIB_NAME "paddle_master") +go_library(${MASTER_LIB_NAME} SHARED) + +if(PROJ_ROOT) + add_custom_command(OUTPUT ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so + COMMAND rm ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.h + COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/lib${MASTER_LIB_NAME}.so ${PROJ_ROOT}/python/paddle/v2/master/ + DEPENDS ${MASTER_LIB_NAME}) + add_custom_target(paddle_master_shared ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/master/lib${MASTER_LIB_NAME}.so) +endif(PROJ_ROOT) diff --git a/go/master/python/.gitignore b/go/master/python/.gitignore deleted file mode 100644 index 704d30751..000000000 --- a/go/master/python/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.whl diff --git a/go/master/python/build.sh b/go/master/python/build.sh deleted file mode 100755 index e3dbd7b0b..000000000 --- a/go/master/python/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -go build -buildmode=c-shared ../c && rm c.h && mv c paddle_master/libmaster.so -pip wheel . diff --git a/go/master/python/setup.py b/go/master/python/setup.py deleted file mode 100644 index d7b6e9eca..000000000 --- a/go/master/python/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import setup, Distribution - - -class BinaryDistribution(Distribution): - def has_ext_modules(foo): - return True - - -setup( - name='paddle_master', - version='0.1', - description='The client of the master server of PaddlePaddle.', - url='https://github.com/PaddlePaddle/Paddle/go/master/python', - author='PaddlePaddle Authors', - author_email='paddle-dev@baidu.com', - license='Apache 2.0', - packages=['paddle_master'], - package_data={'master': ['libmaster.so'], }, - distclass=BinaryDistribution) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 3640dd3a7..345d2fa65 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -15,10 +15,10 @@ set(PY_FILES paddle/__init__.py configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py) -add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp + add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp - DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies}) + DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies} paddle_master_shared) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) diff --git a/python/paddle/v2/__init__.py b/python/paddle/v2/__init__.py index b9d0a7f29..102331c0b 100644 --- a/python/paddle/v2/__init__.py +++ b/python/paddle/v2/__init__.py @@ -26,6 +26,7 @@ import evaluator from . import dataset from . import reader from . import plot +from . import master import attr import op import pooling @@ -37,9 +38,26 @@ import plot import image __all__ = [ - 'optimizer', 'layer', 'activation', 'parameters', 'init', 'trainer', - 'event', 'data_type', 'attr', 'pooling', 'data_feeder', 'dataset', 'reader', - 'topology', 'networks', 'infer', 'plot', 'evaluator', 'image' + 'optimizer', + 'layer', + 'activation', + 'parameters', + 'init', + 'trainer', + 'event', + 'data_type', + 'attr', + 'pooling', + 'data_feeder', + 'dataset', + 'reader', + 'topology', + 'networks', + 'infer', + 'plot', + 'evaluator', + 'image', + 'master', ] diff --git a/go/master/python/paddle_master/.gitignore b/python/paddle/v2/master/.gitignore similarity index 64% rename from go/master/python/paddle_master/.gitignore rename to python/paddle/v2/master/.gitignore index 4cd992125..a3ac6e1a3 100644 --- a/go/master/python/paddle_master/.gitignore +++ b/python/paddle/v2/master/.gitignore @@ -1,2 +1,3 @@ +*.whl *.so *.pyc diff --git a/go/master/python/paddle_master/__init__.py b/python/paddle/v2/master/__init__.py similarity index 100% rename from go/master/python/paddle_master/__init__.py rename to python/paddle/v2/master/__init__.py diff --git a/go/master/python/paddle_master/client.py b/python/paddle/v2/master/client.py similarity index 93% rename from go/master/python/paddle_master/client.py rename to python/paddle/v2/master/client.py index 2dfcb3990..de8e9bb88 100644 --- a/go/master/python/paddle_master/client.py +++ b/python/paddle/v2/master/client.py @@ -1,7 +1,7 @@ import ctypes import os -path = os.path.join(os.path.dirname(__file__), "libmaster.so") +path = os.path.join(os.path.dirname(__file__), "libpaddle_master.so") lib = ctypes.cdll.LoadLibrary(path) diff --git a/python/setup.py.in b/python/setup.py.in index 93724f918..8fe1cfd8b 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,6 +1,5 @@ from setuptools import setup - packages=['paddle', 'paddle.proto', 'paddle.trainer', @@ -9,7 +8,8 @@ packages=['paddle', 'paddle.v2', 'paddle.v2.dataset', 'paddle.v2.reader', - 'paddle.v2.plot'] + 'paddle.v2.plot', + 'paddle.v2.master'] setup_requires=["requests", "numpy", @@ -25,7 +25,8 @@ setup(name='paddle', description='Parallel Distributed Deep Learning', install_requires=setup_requires, packages=packages, + package_data={'paddle.v2.master': ['libpaddle_master.so'], }, package_dir={ '': '${CMAKE_CURRENT_SOURCE_DIR}' - } + }, ) -- GitLab