Merge pull request #1538 from zh794390558/cmake

[speechx] libsndfile , openfst, openblas cmake

Merge pull request #1538 from zh794390558/cmake
[speechx] libsndfile , openfst, openblas cmake
1a1ba2b4 · YangZhou · GitHub · 6a32a0bf · 3887a9b2 · 1a1ba2b4
11 changed file
--- a/speechx/CMakeLists.txt
+++ b/speechx/CMakeLists.txt
@@ -24,7 +24,10 @@ get_filename_component(fc_patch "fc_patch" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR
 set(FETCHCONTENT_BASE_DIR ${fc_patch})

 # compiler option
+# Keep the same with openfst, -fPIC or -fpic
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g")
+SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O0 -Wall -g -ggdb")
+SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} --std=c++14 -pthread -fPIC -O3 -Wall")

 ###############################################################################
 # Option Configurations
@@ -57,9 +60,10 @@ include(absl)
 include(libsndfile)

 # boost
+# include(boost) # not work
 set(boost_SOURCE_DIR ${fc_patch}/boost-src)
 set(BOOST_ROOT ${boost_SOURCE_DIR})
-#find_package(boost REQUIRED PATHS ${BOOST_ROOT})
+# #find_package(boost REQUIRED PATHS ${BOOST_ROOT})

 # Eigen
 include(eigen)
@@ -69,27 +73,13 @@ find_package(Eigen3 REQUIRED)
 include(kenlm)
 add_dependencies(kenlm eigen boost)

+#openblas
+include(openblas)

 # openfst
-set(openfst_SOURCE_DIR ${fc_patch}/openfst-src)
-set(openfst_BINARY_DIR ${fc_patch}/openfst-build)
-set(openfst_PREFIX_DIR ${fc_patch}/openfst-subbuild/openfst-populate-prefix)
-ExternalProject_Add(openfst
-  URL               https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip
-  URL_HASH          SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6
-  #PREFIX            ${openfst_PREFIX_DIR} 
-  SOURCE_DIR        ${openfst_SOURCE_DIR}
-  BINARY_DIR        ${openfst_BINARY_DIR}
-  CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR}
-                      "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}"
-                      "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}"
-                      "LIBS=-lgflags_nothreads -lglog -lpthread"
-  COMMAND           ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
-  BUILD_COMMAND     make -j 4
-)
+include(openfst)
 add_dependencies(openfst gflags glog)
-link_directories(${openfst_PREFIX_DIR}/lib)
-include_directories(${openfst_PREFIX_DIR}/include)
+

 # paddle lib
 set(paddle_SOURCE_DIR ${fc_patch}/paddle-lib)
@@ -133,30 +123,7 @@ set(DEPS ${DEPS}
      glog gflags protobuf xxhash cryptopp
      ${EXTERNAL_LIB})

-#openblas
-set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/openblas-install)
-#set(OpenBLAS_INSTALL_PREFIX ${fc_patch}/OpenBLAS)
-#set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src)
-#set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix)
-#ExternalProject_Add(
-  #OpenBLAS
-  #GIT_REPOSITORY https://github.com/xianyi/OpenBLAS
-  #GIT_TAG        1ef97c470ccf23d06d3bb0597c52df4a2e9093b9
-  #GIT_SHALLOW TRUE
-  #GIT_PROGRESS TRUE
-  #PREFIX      ${OpenBLAS_PREFIX}
-  #SOURCE_DIR  ${OpenBLAS_SOURCE_DIR}
-  #BUILD_IN_SOURCE TRUE
-  #CONFIGURE_COMMAND ""
-  #BUILD_COMMAND make
-  #INSTALL_COMMAND make PREFIX=${OpenBLAS_INSTALL_PREFIX} install
-  #UPDATE_DISCONNECTED TRUE
-#)
-link_directories(${OpenBLAS_INSTALL_PREFIX}/lib)
-include_directories(${OpenBLAS_INSTALL_PREFIX}/include)
-
-# gfortan dir in the docker.
-link_directories(/usr/local/gcc-8.2/lib64)
+

 ###############################################################################
 # Add local library

--- a/speechx/build.sh
+++ b/speechx/build.sh
@@ -16,11 +16,13 @@ if [ ! -d ${boost_SOURCE_DIR} ]; then wget -c https://boostorg.jfrog.io/artifact
  echo -e "\n"
 fi

+rm -rf build
 mkdir -p build
 cd build

-cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR} --target clean
+cmake .. -DBOOST_ROOT:STRING=${boost_SOURCE_DIR}
+#cmake .. 

-make
+make -j1

 cd -
--- a/speechx/cmake/EnableCMP0048.cmake
+++ b/speechx/cmake/EnableCMP0048.cmake
-cmake_policy(SET CMP0048 NEW)
+cmake_policy(SET CMP0048 NEW)
\ No newline at end of file
--- a/speechx/cmake/external/absl.cmake
+++ b/speechx/cmake/external/absl.cmake
 include(FetchContent)

+
+set(BUILD_SHARED_LIBS OFF) # up to you
+set(BUILD_TESTING OFF) # to disable abseil test, or gtest will fail.
+set(ABSL_ENABLE_INSTALL ON) # now you can enable install rules even in subproject...
+
 FetchContent_Declare(
  absl
  GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git"

--- a/speechx/cmake/external/boost.cmake
+++ b/speechx/cmake/external/boost.cmake
+include(FetchContent)
+set(Boost_DEBUG ON)
+
+set(Boost_PREFIX_DIR ${fc_patch}/boost)
+set(Boost_SOURCE_DIR ${fc_patch}/boost-src)
+
+FetchContent_Declare(
+  Boost
+  URL      https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz
+  URL_HASH SHA256=aeb26f80e80945e82ee93e5939baebdca47b9dee80a07d3144be1e1a6a66dd6a
+  PREFIX            ${Boost_PREFIX_DIR}
+  SOURCE_DIR        ${Boost_SOURCE_DIR}
+)
+
+execute_process(COMMAND bootstrap.sh WORKING_DIRECTORY ${Boost_SOURCE_DIR})
+execute_process(COMMAND b2 WORKING_DIRECTORY ${Boost_SOURCE_DIR})
+
+FetchContent_MakeAvailable(Boost)
+
+message(STATUS "boost src dir: ${Boost_SOURCE_DIR}")
+message(STATUS "boost inc dir: ${Boost_INCLUDE_DIR}")
+message(STATUS "boost bin dir: ${Boost_BINARY_DIR}")
+
+set(BOOST_ROOT ${Boost_SOURCE_DIR})
+message(STATUS "boost root dir: ${BOOST_ROOT}")
+
+include_directories(${Boost_SOURCE_DIR})
\ No newline at end of file
--- a/speechx/cmake/external/glog.cmake
+++ b/speechx/cmake/external/glog.cmake
 include(FetchContent)
-
 FetchContent_Declare(
  glog
  URL      https://github.com/google/glog/archive/v0.4.0.zip
  URL_HASH SHA256=9e1b54eb2782f53cd8af107ecf08d2ab64b8d0dc2b7f5594472f3bd63ca85cdc
 )
 FetchContent_MakeAvailable(glog)
-
 include_directories(${glog_BINARY_DIR} ${glog_SOURCE_DIR}/src)
--- a/speechx/cmake/external/gtest.cmake
+++ b/speechx/cmake/external/gtest.cmake
@@ -5,4 +5,5 @@ FetchContent_Declare(
  URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91
 )
 FetchContent_MakeAvailable(gtest)
+
 include_directories(${gtest_BINARY_DIR} ${gtest_SOURCE_DIR}/src)
\ No newline at end of file
--- a/speechx/cmake/external/kenlm.cmake
+++ b/speechx/cmake/external/kenlm.cmake
 include(FetchContent)
-
 FetchContent_Declare(
  kenlm
  GIT_REPOSITORY "https://github.com/kpu/kenlm.git"
  GIT_TAG "df2d717e95183f79a90b2fa6e4307083a351ca6a"
 )
-
 # https://github.com/kpu/kenlm/blob/master/cmake/modules/FindEigen3.cmake
 set(EIGEN3_INCLUDE_DIR ${Eigen3_SOURCE_DIR})
-
 FetchContent_MakeAvailable(kenlm)
-
 include_directories(${kenlm_SOURCE_DIR})
\ No newline at end of file
--- a/speechx/cmake/external/libsndfile.cmake
+++ b/speechx/cmake/external/libsndfile.cmake
@@ -11,8 +11,8 @@ else()
  set(FETCHCONTENT_SOURCE_DIR_LIBSNDFILE "")
 endif()

-set(LIBSNDFILE_GIT_REPO "https://github.com/erikd/libsndfile" CACHE STRING "libsndfile git repository url" FORCE)
-set(LIBSNDFILE_GIT_TAG c11deaa04ec84161996824061f6d705970972e2e CACHE STRING "libsndfile git tag" FORCE)
+set(LIBSNDFILE_GIT_REPO "https://github.com/libsndfile/libsndfile.git" CACHE STRING "libsndfile git repository url" FORCE)
+set(LIBSNDFILE_GIT_TAG 1.0.31 CACHE STRING "libsndfile git tag" FORCE)

 FetchContent_Declare(libsndfile
      GIT_REPOSITORY    ${LIBSNDFILE_GIT_REPO}
@@ -48,7 +48,7 @@ function(libsndfile_build)
  # finally we include libsndfile itself
  add_subdirectory(${libsndfile_SOURCE_DIR} ${libsndfile_BINARY_DIR} EXCLUDE_FROM_ALL)
  # copying .hh for c++ support
-  file(COPY "${libsndfile_SOURCE_DIR}/src/sndfile.hh" DESTINATION ${LIBSNDFILE_INCLUDE_DIR})
+  #file(COPY "${libsndfile_SOURCE_DIR}/src/sndfile.hh" DESTINATION ${LIBSNDFILE_INCLUDE_DIR})
 endfunction()

 libsndfile_build()

--- a/speechx/cmake/external/openblas.cmake
+++ b/speechx/cmake/external/openblas.cmake
+include(FetchContent)
+
+set(OpenBLAS_SOURCE_DIR ${fc_patch}/OpenBLAS-src)
+set(OpenBLAS_PREFIX ${fc_patch}/OpenBLAS-prefix)
+
+# ######################################################################################################################
+# OPENBLAS  https://github.com/lattice/quda/blob/develop/CMakeLists.txt#L575
+# ######################################################################################################################
+enable_language(Fortran)
+#TODO: switch to CPM
+include(GNUInstallDirs)
+ExternalProject_Add(
+    OPENBLAS
+    GIT_REPOSITORY https://github.com/xianyi/OpenBLAS.git
+    GIT_TAG v0.3.10
+    GIT_SHALLOW YES
+    PREFIX ${OpenBLAS_PREFIX}
+    SOURCE_DIR  ${OpenBLAS_SOURCE_DIR}
+    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> 
+    CMAKE_GENERATOR "Unix Makefiles")
+# https://cmake.org/cmake/help/latest/module/ExternalProject.html?highlight=externalproject_get_property#external-project-definition
+ExternalProject_Get_Property(OPENBLAS INSTALL_DIR)
+set(OpenBLAS_INSTALL_PREFIX ${INSTALL_DIR})
+add_library(openblas STATIC IMPORTED)
+add_dependencies(openblas OPENBLAS)
+set_target_properties(openblas PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES Fortran)
+# ${CMAKE_INSTALL_LIBDIR}  lib
+set_target_properties(openblas PROPERTIES IMPORTED_LOCATION ${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libopenblas.a)
+
+
+# https://cmake.org/cmake/help/latest/command/install.html?highlight=cmake_install_libdir#installing-targets
+# ${CMAKE_INSTALL_LIBDIR}  lib
+# ${CMAKE_INSTALL_INCLUDEDIR}  include
+link_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
+include_directories(${OpenBLAS_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR})
\ No newline at end of file
--- a/speechx/cmake/external/openfst.cmake
+++ b/speechx/cmake/external/openfst.cmake
+include(FetchContent)
+set(openfst_SOURCE_DIR ${fc_patch}/openfst-src)
+set(openfst_BINARY_DIR ${fc_patch}/openfst-build)
+
+ExternalProject_Add(openfst
+  URL               https://github.com/mjansche/openfst/archive/refs/tags/1.7.2.zip
+  URL_HASH          SHA256=ffc56931025579a8af3515741c0f3b0fc3a854c023421472c07ca0c6389c75e6
+#   #PREFIX            ${openfst_PREFIX_DIR} 
+#   SOURCE_DIR        ${openfst_SOURCE_DIR}
+#   BINARY_DIR        ${openfst_BINARY_DIR}
+  CONFIGURE_COMMAND ${openfst_SOURCE_DIR}/configure --prefix=${openfst_PREFIX_DIR}
+                      "CPPFLAGS=-I${gflags_BINARY_DIR}/include -I${glog_SOURCE_DIR}/src -I${glog_BINARY_DIR}"
+                      "LDFLAGS=-L${gflags_BINARY_DIR} -L${glog_BINARY_DIR}"
+                      "LIBS=-lgflags_nothreads -lglog -lpthread"
+  COMMAND           ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/patch/openfst ${openfst_SOURCE_DIR}
+  BUILD_COMMAND     make -j 4
+)
+link_directories(${openfst_PREFIX_DIR}/lib)
+include_directories(${openfst_PREFIX_DIR}/include)