From 7355d41834928eeb341180fef50acdb4620aa8f1 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 14 Mar 2019 14:55:26 +0800 Subject: [PATCH] 1. Add imperative gperf profiler 2. Add binutils 2.27 in manylinux support test=develop --- CMakeLists.txt | 12 +--- paddle/fluid/CMakeLists.txt | 2 +- paddle/fluid/imperative/profiler.cc | 65 ++++++++++++++++++++++ paddle/fluid/imperative/profiler.h | 25 +++++++++ paddle/fluid/inference/CMakeLists.txt | 51 +++++++++-------- paddle/fluid/pybind/pybind.cc | 5 +- python/paddle/fluid/imperative/profiler.py | 30 ++++++++++ tools/manylinux1/build_scripts/build.sh | 6 ++ 8 files changed, 156 insertions(+), 40 deletions(-) create mode 100644 paddle/fluid/imperative/profiler.cc create mode 100644 paddle/fluid/imperative/profiler.h create mode 100644 python/paddle/fluid/imperative/profiler.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ce06c73bdcf..6bb0e5f51f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,8 @@ message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}, version: " "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: " "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}") +message(STATUS "AR tools: ${CMAKE_AR}") + if(WIN32) set(CMAKE_SUPPRESS_REGENERATION ON) set(CMAKE_STATIC_LIBRARY_PREFIX lib) @@ -37,16 +39,6 @@ if(WIN32) set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}") -# else() - # set(CMAKE_C_ARCHIVE_CREATE " --target elf64-x86-64 cr ") - # set(CMAKE_C_ARCHIVE_APPEND " --target elf64-x86-64 r ") - # # set(CMAKE_C_ARCHIVE_FINISH " --enable-64-bit-archive ") - # set(CMAKE_CXX_ARCHIVE_CREATE ${CMAKE_C_ARCHIVE_CREATE}) - # set(CMAKE_CXX_ARCHIVE_APPEND ${CMAKE_C_ARCHIVE_APPEND}) - # # set(CMAKE_CXX_ARCHIVE_FINISH ${CMAKE_C_ARCHIVE_FINISH}) - # set(CMAKE_Fortran_ARCHIVE_CREATE ${CMAKE_C_ARCHIVE_CREATE}) - # set(CMAKE_Fortran_ARCHIVE_APPEND ${CMAKE_C_ARCHIVE_APPEND}) - # # set(CMAKE_Fortran_ARCHIVE_FINISH ${CMAKE_C_ARCHIVE_FINISH}) endif(WIN32) find_package(CUDA QUIET) diff --git a/paddle/fluid/CMakeLists.txt b/paddle/fluid/CMakeLists.txt index 6e951e42a0f..595454e90b9 100644 --- a/paddle/fluid/CMakeLists.txt +++ b/paddle/fluid/CMakeLists.txt @@ -9,4 +9,4 @@ add_subdirectory(pybind) # NOTE: please add subdirectory inference at last. add_subdirectory(inference) -#add_subdirectory(train) +add_subdirectory(train) diff --git a/paddle/fluid/imperative/profiler.cc b/paddle/fluid/imperative/profiler.cc new file mode 100644 index 00000000000..828c36c5aef --- /dev/null +++ b/paddle/fluid/imperative/profiler.cc @@ -0,0 +1,65 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/imperative/profiler.h" + +#ifdef WITH_GPERFTOOLS +#include "gperftools/profiler.h" +#endif +#include +#include +#include // NOLINT +#include // NOLINT + +DEFINE_string( + tracer_profile_fname, "xxgperf", + "Profiler filename for imperative tracer, which generated by gperftools." + "Only valid when compiled `WITH_PROFILER=ON`. Empty if disable."); + +namespace paddle { +namespace imperative { + +static std::once_flag gTracerProfileOnce; +#ifdef WITH_GPERFTOOLS +static bool gTracerProfilerStarted = false; +#endif + +void StartProfile() { + LOG(ERROR) << "XX " << FLAGS_tracer_profile_fname; + if (!FLAGS_tracer_profile_fname.empty()) { + std::call_once(gTracerProfileOnce, [] { +#ifdef WITH_GPERFTOOLS + ProfilerStart(FLAGS_tracer_profile_fname.c_str()); + gTracerProfilerStarted = true; + LOG(ERROR) << "YY"; +#else + LOG(WARNING) << "Paddle is not compiled with gperftools. " + "FLAGS_tracer_profile_fname will be ignored"; +#endif + }); + } +} + +void StopProfile() { + LOG(ERROR) << "ZZ " << FLAGS_tracer_profile_fname; +#ifdef WITH_GPERFTOOLS + ProfilerFlush(); +#else + LOG(WARNING) << "Paddle is not compiled with gperftools. " + "FLAGS_tracer_profile_fname will be ignored"; +#endif +} + +} // namespace imperative +} // namespace paddle diff --git a/paddle/fluid/imperative/profiler.h b/paddle/fluid/imperative/profiler.h new file mode 100644 index 00000000000..d52aeed4e81 --- /dev/null +++ b/paddle/fluid/imperative/profiler.h @@ -0,0 +1,25 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +namespace paddle { +namespace imperative { + +extern void StartProfile(); + +extern void StopProfile(); + +} // namespace imperative +} // namespace paddle diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index a89dc3caf55..d27ef8fe3c3 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -25,13 +25,12 @@ if (WIN32) list(APPEND fluid_third_partys gflags glog protobuf cblas) endif(WIN32) -# # paddle_fluid_origin exclude inference api interface -# if(WIN32) - # sep_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api) -# else(WIN32) - # set(CMAKE_C_ARCHIVE_CREATE " --target elf64-x86-64 cr paddle_fluid_origin ${fluid_modules} paddle_fluid_api") - # cc_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api) -# endif(WIN32) +# paddle_fluid_origin exclude inference api interface +if(WIN32) + sep_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api) +else(WIN32) + cc_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api) +endif(WIN32) add_subdirectory(api) @@ -41,19 +40,19 @@ set(SHARED_INFERENCE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/api/analysis_predictor.cc ${CMAKE_CURRENT_SOURCE_DIR}/api/details/zero_copy_tensor.cc) -# if(WIN32) - # sep_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} zero_copy_tensor reset_tensor_array - # analysis_config paddle_pass_builder) -# else(WIN32) - # cc_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} - # zero_copy_tensor reset_tensor_array analysis_config paddle_pass_builder) -# endif(WIN32) +if(WIN32) + sep_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} zero_copy_tensor reset_tensor_array + analysis_config paddle_pass_builder) +else(WIN32) + cc_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} + zero_copy_tensor reset_tensor_array analysis_config paddle_pass_builder) +endif(WIN32) -# if(NOT APPLE) - # # TODO(liuyiqu: Temporarily disable the link flag because it is not support on Mac. - # set(LINK_FLAGS "-Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/paddle_fluid.sym") - # set_target_properties(paddle_fluid PROPERTIES LINK_FLAGS "${LINK_FLAGS}") -# endif() +if(NOT APPLE) + # TODO(liuyiqu: Temporarily disable the link flag because it is not support on Mac. + set(LINK_FLAGS "-Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/paddle_fluid.sym") + set_target_properties(paddle_fluid PROPERTIES LINK_FLAGS "${LINK_FLAGS}") +endif() # Create shared library if(WIN32) @@ -87,10 +86,10 @@ if(NOT APPLE AND NOT WIN32) add_custom_target(check_symbol ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/.check_symbol") endif() -# if(WITH_TESTING) - # # tests/book depends the models that generated by python/paddle/fluid/tests/book - # add_subdirectory(tests/book) - # if(WITH_INFERENCE_API_TEST) - # add_subdirectory(tests/api) - # endif() -# endif() +if(WITH_TESTING) + # tests/book depends the models that generated by python/paddle/fluid/tests/book + add_subdirectory(tests/book) + if(WITH_INFERENCE_API_TEST) + add_subdirectory(tests/api) + endif() +endif() diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 7a6a5b8645d..df6e2fbab64 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -149,11 +149,10 @@ PYBIND11_MODULE(core, m) { m.def("print_mem_usage", []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); }); - m.def("start_imperative_profiler", + m.def("start_imperative_gperf_profiler", []() { imperative::StartProfile(); }); - m.def("stop_imperative_profiler", - []() { imperative::StopProfile(); }); + m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); }); py::class_(m, "VarBase", R"DOC()DOC") .def( diff --git a/python/paddle/fluid/imperative/profiler.py b/python/paddle/fluid/imperative/profiler.py new file mode 100644 index 00000000000..04c865500bb --- /dev/null +++ b/python/paddle/fluid/imperative/profiler.py @@ -0,0 +1,30 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +from .. import core + +__all__ = [ + 'start_gperf_profiler', + 'stop_gperf_profiler', +] + + +def start_gperf_profiler(): + core.start_imperative_gperf_profiler() + + +def stop_gperf_profiler(): + core.stop_imperative_gperf_profiler() diff --git a/tools/manylinux1/build_scripts/build.sh b/tools/manylinux1/build_scripts/build.sh index 1b0059a8c69..3be94a42d53 100644 --- a/tools/manylinux1/build_scripts/build.sh +++ b/tools/manylinux1/build_scripts/build.sh @@ -153,3 +153,9 @@ done # Restore LD_LIBRARY_PATH LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}" + +# According to ar issues: https://lists.gnu.org/archive/html/bug-binutils/2016-05/msg00211.html +# we should install new version ar with 64-bit supported here +wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz +tar xzf binutils-2.27.tar.gz && cd binutils-2.27 +./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install -- GitLab