using glide for go package vendor

155e40ef · wuyi05 · 4db23bba · 155e40ef · 155e40ef · 155e40ef
8 changed file
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,6 +48,7 @@ option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
 option(ON_TRAVIS        "Exclude special unit test on Travis CI"        OFF)
 option(WITH_C_API       "Compile PaddlePaddle with C-API(Prediction)"   OFF)
 option(WITH_GOLANG      "Compile PaddlePaddle with GOLANG"              OFF)
+option(GLIDE_INSTALL    "Download and install go dependencies "         ON)

 # CMAKE_BUILD_TYPE
 if(NOT CMAKE_BUILD_TYPE)
@@ -131,8 +132,7 @@ add_subdirectory(paddle)
 add_subdirectory(python)

 if(WITH_GOLANG)
-    #TODO (add go/master/c back when fixed)
-    add_subdirectory(go/pserver/cclient)
+    add_subdirectory(go)
 endif(WITH_GOLANG)

 if(WITH_DOC)

--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -17,7 +17,7 @@
 # generic.cmake defines CMakes functions that look like Bazel's
 # building rules (https://bazel.build/).
 #
-# 
+#
 # -------------------------------------------
 #     C++        CUDA C++       Go
 # -------------------------------------------
@@ -25,51 +25,51 @@
 # cc_binary     nv_binary    go_binary
 # cc_test       nv_test      go_test
 # -------------------------------------------
-# 
+#
 # To build a static library example.a from example.cc using the system
 #  compiler (like GCC):
-# 
+#
 #   cc_library(example SRCS example.cc)
-# 
+#
 # To build a static library example.a from multiple source files
 # example{1,2,3}.cc:
-# 
+#
 #   cc_library(example SRCS example1.cc example2.cc example3.cc)
-# 
+#
 # To build a shared library example.so from example.cc:
-# 
+#
 #   cc_library(example SHARED SRCS example.cc)
-# 
+#
 # To build a library using Nvidia's NVCC from .cu file(s), use the nv_
 # prefixed version:
-# 
+#
 #   nv_library(example SRCS example.cu)
-# 
+#
 # To specify that a library new_example.a depends on other libraies:
-# 
+#
 #   cc_library(new_example SRCS new_example.cc DEPS example)
-# 
+#
 # Static libraries can be composed of other static libraries:
-# 
+#
 #   cc_library(composed DEPS dependent1 dependent2 dependent3)
-# 
+#
 # To build an executable binary file from some source files and
 # dependent libraries:
-# 
+#
 #   cc_binary(example SRCS main.cc something.cc DEPS example1 example2)
-# 
+#
 # To build an executable binary file using NVCC, use the nv_ prefixed
 # version:
-# 
+#
 #   nv_binary(example SRCS main.cc something.cu DEPS example1 example2)
-# 
+#
 # To build a unit test binary, which is an executable binary with
 # GoogleTest linked:
-# 
+#
 #   cc_test(example_test SRCS example_test.cc DEPS example)
-# 
+#
 # To build a unit test binary using NVCC, use the nv_ prefixed version:
-# 
+#
 #   nv_test(example_test SRCS example_test.cu DEPS example)
 #
 # It is pretty often that executable and test binaries depend on
@@ -256,6 +256,8 @@ endfunction(nv_test)
 set(GOPATH "${CMAKE_CURRENT_BINARY_DIR}/go")
 file(MAKE_DIRECTORY ${GOPATH})
 set(PADDLE_IN_GOPATH "${GOPATH}/src/github.com/PaddlePaddle/Paddle")
+file(MAKE_DIRECTORY "${PADDLE_IN_GOPATH}")
+set(PADDLE_GO_SRC "${CMAKE_SOURCE_DIR}/go")

 function(go_library TARGET_NAME)
  set(options STATIC static SHARED shared)
@@ -280,7 +282,7 @@ function(go_library TARGET_NAME)
    add_library(${TARGET_NAME} STATIC ${dummyfile})
  endif()
  if(go_library_DEPS)
-    add_dependencies(${TARGET_NAME} ${go_library_DEPS})
+    add_dependencies(${TARGET_NAME} ${go_library_DEPS} paddle_go_path_link)
  endif(go_library_DEPS)

  # we need to symlink Paddle directory into GOPATH. If we
@@ -289,19 +291,23 @@ function(go_library TARGET_NAME)
  # without the changes in our current Paddle repo that we
  # want to build.
  file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
+  string(REPLACE "${PADDLE_GO_SRC}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR})
  add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
    COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
    # Symlink Paddle directory into GOPATH
    COMMAND mkdir -p ${PADDLE_IN_GOPATH}
-    COMMAND rm -rf ${PADDLE_IN_GOPATH}                                                                                                                                         
+    COMMAND rm -rf ${PADDLE_IN_GOPATH}
    COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
-    # Automatically get all dependencies specified in the source code                                                                                                                                 
-    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./...
+    WORKING_DIRECTORY ${PADDLE_GO_SRC})
+  add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
+    # Automatically get all dependencies specified in the source code
+    #COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} get -d ./...
    # Golang build source code
    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
    -o "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}"
-    ${GO_SOURCE}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    "./${CMAKE_CURRENT_SOURCE_REL_DIR}/${GO_SOURCE}"
+    # must run under GOPATH
+  WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go")
 endfunction(go_library)

 function(go_binary TARGET_NAME)
@@ -309,12 +315,20 @@ function(go_binary TARGET_NAME)
  set(oneValueArgs "")
  set(multiValueArgs SRCS DEPS)
  cmake_parse_arguments(go_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  string(REPLACE "${PADDLE_GO_SRC}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+  add_custom_command(OUTPUT ${TARGET_NAME}_link
+    # Symlink Paddle directory into GOPATH
+    COMMAND mkdir -p ${PADDLE_IN_GOPATH}
+    COMMAND rm -rf ${PADDLE_IN_GOPATH}
+    COMMAND ln -sf ${CMAKE_SOURCE_DIR} ${PADDLE_IN_GOPATH}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+
  add_custom_command(OUTPUT ${TARGET_NAME}_timestamp
    COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
    -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
    ${go_library_SRCS}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_binary_DEPS})
+  WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go/${CMAKE_CURRENT_SOURCE_REL_DIR}")
+  add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_link ${TARGET_NAME}_timestamp ${go_binary_DEPS})
  install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin)
 endfunction(go_binary)


--- a/go/.gitignore
+++ b/go/.gitignore
+vendor/
--- a/go/CMakeLists.txt
+++ b/go/CMakeLists.txt
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# FIXME(typhoonzero): Download glide into cmake build temprary GOPATH
+if(EXISTS $ENV{GOPATH}/bin/glide)
+  set(GLIDE "$ENV{GOPATH}/bin/glide")
+else()
+  message(FATAL_ERROR "no glide executeble found: $ENV{GOPATH}/bin/glide")
+endif()
+
+set(PADDLE_GO_PATH "${CMAKE_SOURCE_DIR}/go")
+
+if (GLIDE_INSTALL)
+  message(STATUS ${PADDLE_GO_PATH})
+  execute_process(COMMAND ${GLIDE} install WORKING_DIRECTORY ${PADDLE_GO_PATH})
+endif()
+
+add_subdirectory(go/pserver/cclient)
+#TODO (add go/master/c back when fixed)
--- a/go/cmd/master/CMakeLists.txt
+++ b/go/cmd/master/CMakeLists.txt
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+go_binary(master)
--- a/go/cmd/pserver/CMakeLists.txt
+++ b/go/cmd/pserver/CMakeLists.txt
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+go_binary(pserver)
--- a/go/glide.lock
+++ b/go/glide.lock
+hash: b8f18ce6784bd3fadd9fed0b8443e7b658234ea785ae1f220723ae2c1f652aa7
+updated: 2017-06-27T14:05:48.925262819+08:00
+imports:
+- name: github.com/coreos/etcd
+  version: 61fc123e7a8b14a0a258aa3f5c4159861b1ec2e7
+  subpackages:
+  - auth/authpb
+  - clientv3
+  - clientv3/concurrency
+  - etcdserver/api/v3rpc/rpctypes
+  - etcdserver/etcdserverpb
+  - mvcc/mvccpb
+- name: github.com/golang/protobuf
+  version: 4bd1920723d7b7c925de087aa32e2187708897f7
+  subpackages:
+  - jsonpb
+  - proto
+- name: github.com/golang/snappy
+  version: 553a641470496b2327abcac10b36396bd98e45c9
+- name: github.com/namsral/flag
+  version: 71ceffbeb0ba60fccc853971bb3ed4d7d90bfd04
+- name: github.com/PaddlePaddle/recordio
+  version: edfb82af0739c84f241c87390ec5649c7b28c129
+- name: github.com/sirupsen/logrus
+  version: 202f25545ea4cf9b191ff7f846df5d87c9382c2b
+- name: golang.org/x/net
+  version: c8c74377599bd978aee1cf3b9b63a8634051cec2
+  subpackages:
+  - context
+  - http2
+  - http2/hpack
+  - idna
+  - internal/timeseries
+  - lex/httplex
+  - trace
+- name: golang.org/x/sys
+  version: f7928cfef4d09d1b080aa2b6fd3ca9ba1567c733
+  subpackages:
+  - unix
+- name: golang.org/x/text
+  version: 4e9ab9ee170f2a39bd66c92b3e0a47ff47a4bc77
+  subpackages:
+  - secure/bidirule
+  - transform
+  - unicode/bidi
+  - unicode/norm
+- name: google.golang.org/grpc
+  version: 8050b9cbc271307e5a716a9d782803d09b0d6f2d
+  subpackages:
+  - codes
+  - credentials
+  - grpclog
+  - internal
+  - keepalive
+  - metadata
+  - naming
+  - peer
+  - stats
+  - tap
+  - transport
+testImports: []
--- a/go/glide.yaml
+++ b/go/glide.yaml
+package: github.com/PaddlePaddle/Paddle/go
+import:
+- package: github.com/PaddlePaddle/recordio
+- package: github.com/coreos/etcd
+  version: ^3.2.1
+  subpackages:
+  - clientv3
+  - clientv3/concurrency
+- package: github.com/namsral/flag
+  version: ^1.7.4-pre
+- package: github.com/sirupsen/logrus
+  version: ^1.0.0