From c38c7c56196978db5ac970a606ebcc4b9aa9c7a9 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Tue, 2 Apr 2019 10:09:21 +0800 Subject: [PATCH] Model data cryption link all lib (#16555) * link the libwbaes.so into paddle * polish detail, test=develop * try fix mac_pr_ci error, test=develop * add compile option, test=develop * fix ci error, test=develop * ignore failed to find mac lib, test=develop * change cdn to bj, cdn can't get the latest version * trigger ci, test=develop * temporary delete win32 lib linking, test=develop * change https to http, test=develop * turn compile option on to off * turn compile option off to on, test=develop * try lib compiled by gcc4.8, test=develop * update lib version, test=develop * link other lib, test=develop * add setup config * delete false, test=develop * delete no_soname, test=develop * recover so name set * fix, test=develop * adjust make config, test=develop * remove link to wbaes, test=develop * remove useless define, test=develop --- CMakeLists.txt | 2 + cmake/configure.cmake | 4 ++ cmake/external/wbaes.cmake | 71 +++++++++++++++++++ cmake/generic.cmake | 8 +++ cmake/inference_lib.cmake | 8 +++ paddle/fluid/platform/dynload/CMakeLists.txt | 3 + .../fluid/platform/dynload/dynamic_loader.cc | 12 ++++ .../fluid/platform/dynload/dynamic_loader.h | 1 + paddle/fluid/platform/dynload/wbaes.cc | 34 +++++++++ paddle/fluid/platform/dynload/wbaes.h | 63 ++++++++++++++++ python/setup.py.in | 4 ++ 11 files changed, 210 insertions(+) create mode 100644 cmake/external/wbaes.cmake create mode 100644 paddle/fluid/platform/dynload/wbaes.cc create mode 100644 paddle/fluid/platform/dynload/wbaes.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ad69738e..9b77659f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,7 @@ option(WITH_INFERENCE_API_TEST "Test fluid inference high-level api interface" option(WITH_SYSTEM_BLAS "Use system blas library" OFF) option(PY_VERSION "Compile PaddlePaddle with python3 support" ${PY_VERSION}) option(WITH_FAST_MATH "Make use of fast math library, might affect the precision to some extent" ON) +option(WITH_WBAES "Compile PaddlePaddle with WBAES support" ON) # PY_VERSION if(NOT PY_VERSION) @@ -148,6 +149,7 @@ include(external/dlpack) include(external/snappy) # download snappy include(external/snappystream) # download snappystream include(external/warpctc) # download, build, install warpctc +include(external/wbaes) # download wbaes if (NOT WIN32) # there is no official support of nccl, cupti in windows diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 93d74bb0a..283845541 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -157,3 +157,7 @@ endif(WITH_BRPC_RDMA) if(ON_INFER) add_definitions(-DPADDLE_ON_INFERENCE) endif(ON_INFER) + +if(WITH_WBAES) + add_definitions(-DPADDLE_WITH_WBAES) +endif(WITH_WBAES) diff --git a/cmake/external/wbaes.cmake b/cmake/external/wbaes.cmake new file mode 100644 index 000000000..feda5cb36 --- /dev/null +++ b/cmake/external/wbaes.cmake @@ -0,0 +1,71 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IF(NOT ${WITH_WBAES}) + return() +ENDIF(NOT ${WITH_WBAES}) + +INCLUDE(ExternalProject) +SET(WBAES_DST_DIR "wbaes") +SET(WBAES_INSTALL_ROOT "${THIRD_PARTY_PATH}/install") +SET(WBAES_INSTALL_DIR ${WBAES_INSTALL_ROOT}/${WBAES_DST_DIR}) +SET(WBAES_ROOT ${WBAES_INSTALL_DIR}) +SET(WBAES_INC_DIR ${WBAES_ROOT}/include) +SET(WBAES_LIB_DIR ${WBAES_ROOT}/lib) + +SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${WBAES_ROOT}/lib") +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +IF(APPLE) + SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE) + SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.mac.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) + SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.dylib) + SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dylib) +ELSEIF(WIN32) + SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE) + SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.windows-x64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) + SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.lib) + SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dll) +ELSE() + SET(WBAES_TAG "v1.0.2" CACHE STRING "" FORCE) + SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.linux-x86_64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) + SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.so) + SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.so) +ENDIF() + +SET(WBAES_PROJECT "extern_wbaes") +MESSAGE(STATUS "WBAES_URL: ${WBAES_URL}, WBAES_LIB: ${WBAES_LIB}") +SET(WBAES_SOURCE_DIR "${THIRD_PARTY_PATH}/wbaes") +SET(WBAES_DOWNLOAD_DIR "${WBAES_SOURCE_DIR}/src/${WBAES_PROJECT}") + +ExternalProject_Add( + ${WBAES_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${WBAES_SOURCE_DIR} + URL ${WBAES_URL} + DOWNLOAD_DIR ${WBAES_DOWNLOAD_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + ${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/include ${WBAES_INC_DIR} && + ${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/lib ${WBAES_LIB_DIR} +) + +INCLUDE_DIRECTORIES(${WBAES_INC_DIR}) + +ADD_LIBRARY(wbaes SHARED IMPORTED GLOBAL) +SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_LOCATION ${WBAES_LIB}) +SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_NO_SONAME 1) +ADD_DEPENDENCIES(wbaes ${WBAES_PROJECT}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 6679a09df..19110812c 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -264,6 +264,14 @@ function(cc_library TARGET_NAME) list(REMOVE_ITEM cc_library_DEPS warpctc) add_dependencies(${TARGET_NAME} warpctc) endif() + # Only deps libwbaes.so, not link + if("${cc_library_DEPS};" MATCHES "wbaes;") + list(REMOVE_ITEM cc_library_DEPS wbaes) + if(NOT "${TARGET_NAME}" MATCHES "dynload_wbaes") + list(APPEND cc_library_DEPS dynload_wbaes) + endif() + add_dependencies(${TARGET_NAME} wbaes) + endif() # Only deps libmklml.so, not link if("${cc_library_DEPS};" MATCHES "mklml;") list(REMOVE_ITEM cc_library_DEPS mklml) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index b7c32f80d..2f558bffb 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -170,6 +170,14 @@ copy(snappystream_lib DSTS ${dst_dir} ${dst_dir}/lib DEPS snappystream) +if (WITH_WBAES) + set(dst_dir "${FLUID_INSTALL_DIR}/third_party/install/wbaes") + copy(wbaes_lib + SRCS ${WBAES_INC_DIR} ${WBAES_LIB} + DSTS ${dst_dir} ${dst_dir}/lib + DEPS wbaes) +endif () + set(dst_dir "${FLUID_INSTALL_DIR}/third_party/install/zlib") copy(zlib_lib SRCS ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES} diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt index 07159d4a1..169734379 100644 --- a/paddle/fluid/platform/dynload/CMakeLists.txt +++ b/paddle/fluid/platform/dynload/CMakeLists.txt @@ -17,6 +17,9 @@ if (CUPTI_FOUND) endif(CUPTI_FOUND) nv_library(dynload_cuda SRCS ${CUDA_SRCS} DEPS dynamic_loader) cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc) +if (WITH_WBAES) + cc_library(dynload_wbaes SRCS wbaes.cc DEPS dynamic_loader wbaes) +endif() if (WITH_MKLML) cc_library(dynload_mklml SRCS mklml.cc DEPS dynamic_loader mklml) endif() diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 15d516836..8ac939378 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -48,6 +48,8 @@ DEFINE_string( DEFINE_string(mklml_dir, "", "Specify path for loading libmklml_intel.so."); +DEFINE_string(wbaes_dir, "", "Specify path for loading libwbaes.so."); + namespace paddle { namespace platform { namespace dynload { @@ -246,6 +248,16 @@ void* GetMKLMLDsoHandle() { #endif } +void* GetWBAESDsoHandle() { +#if defined(__APPLE__) || defined(__OSX__) + return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.dylib"); +#elif defined(_WIN32) + return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.dll"); +#else + return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.so"); +#endif +} + } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/fluid/platform/dynload/dynamic_loader.h b/paddle/fluid/platform/dynload/dynamic_loader.h index edb4c649a..5a642967c 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.h +++ b/paddle/fluid/platform/dynload/dynamic_loader.h @@ -32,6 +32,7 @@ void* GetWarpCTCDsoHandle(); void* GetNCCLDsoHandle(); void* GetTensorRtDsoHandle(); void* GetMKLMLDsoHandle(); +void* GetWBAESDsoHandle(); } // namespace dynload } // namespace platform diff --git a/paddle/fluid/platform/dynload/wbaes.cc b/paddle/fluid/platform/dynload/wbaes.cc new file mode 100644 index 000000000..37387b202 --- /dev/null +++ b/paddle/fluid/platform/dynload/wbaes.cc @@ -0,0 +1,34 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PADDLE_WITH_WBAES + +#include "paddle/fluid/platform/dynload/wbaes.h" + +namespace paddle { +namespace platform { +namespace dynload { + +std::once_flag wbaes_dso_flag; +void *wbaes_dso_handle = nullptr; + +#define DEFINE_WRAP(__name) DynLoad__##__name __name + +WBAES_ROUTINE_EACH(DEFINE_WRAP); + +} // namespace dynload +} // namespace platform +} // namespace paddle + +#endif diff --git a/paddle/fluid/platform/dynload/wbaes.h b/paddle/fluid/platform/dynload/wbaes.h new file mode 100644 index 000000000..22400d44e --- /dev/null +++ b/paddle/fluid/platform/dynload/wbaes.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +#pragma once + +#ifdef PADDLE_WITH_WBAES + +#include +#include // NOLINT + +#include "paddle/fluid/platform/dynload/dynamic_loader.h" +#include "paddle/fluid/platform/port.h" + +namespace paddle { +namespace platform { +namespace dynload { + +extern std::once_flag wbaes_dso_flag; +extern void *wbaes_dso_handle; + +/** + * The following macro definition can generate structs + * (for each function) to dynamic load wbaes routine + * via operator overloading. + */ + +#define DYNAMIC_LOAD_WBAES_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> DECLARE_TYPE(__name, args...) { \ + using wbaesFunc = decltype(&::__name); \ + std::call_once(wbaes_dso_flag, []() { \ + wbaes_dso_handle = paddle::platform::dynload::GetWBAESDsoHandle(); \ + }); \ + static void *p_##__name = dlsym(wbaes_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ + }; \ + extern DynLoad__##__name __name + +#define DECLARE_DYNAMIC_LOAD_WBAES_WRAP(__name) DYNAMIC_LOAD_WBAES_WRAP(__name) + +#define WBAES_ROUTINE_EACH(__macro) __macro(GSECF); + +WBAES_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_WBAES_WRAP); + +#undef DYNAMIC_LOAD_WBAES_WRAP + +} // namespace dynload +} // namespace platform +} // namespace paddle + +#endif diff --git a/python/setup.py.in b/python/setup.py.in index eef8afac6..9ab4e9742 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -157,6 +157,10 @@ package_data['paddle.libs']= [] package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name] shutil.copy('${WARPCTC_LIBRARIES}', libs_path) +if '${WITH_WBAES}' == 'ON': + package_data['paddle.libs'] += ['libwbaes' + ext_name] + shutil.copy('${WBAES_SHARED_LIB}', libs_path) + if '${WITH_MKL}' == 'ON': shutil.copy('${MKLML_SHARED_LIB}', libs_path) shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path) -- GitLab