diff --git a/CMakeLists.txt b/CMakeLists.txt index df00e977ebb547980e69ee421779c57717d771a9..bb2ba1ea0c416b00f6a8b9b2f7cf7d6a018c10ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,6 +176,7 @@ include(external/eigen) # download eigen3 include(external/pybind11) # download pybind11 include(external/cares) include(external/cub) +include(external/gzstream) if (NOT WIN32) # there is no official support of snappystream, warpctc, nccl, cupti in windows diff --git a/cmake/external/gzstream.cmake b/cmake/external/gzstream.cmake new file mode 100644 index 0000000000000000000000000000000000000000..f0e3dd8c6aa115be7b43d8d80ebdc2084cbcc23f --- /dev/null +++ b/cmake/external/gzstream.cmake @@ -0,0 +1,47 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +IF(MOBILE_INFERENCE) + return() +ENDIF() + +include (ExternalProject) + +# NOTE: gzstream is needed when linking with ctr reader. + +SET(GZSTREAM_SOURCES_DIR ${THIRD_PARTY_PATH}/gzstream) +SET(GZSTREAM_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gzstream) +SET(GZSTREAM_INCLUDE_DIR "${GZSTREAM_INSTALL_DIR}/include/" CACHE PATH "gzstream include directory." FORCE) + +ExternalProject_Add( + extern_gzstream + GIT_REPOSITORY "https://github.com/kanedo/gzstream.git" + GIT_TAG "" + PREFIX ${GZSTREAM_SOURCES_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND make -j8 + INSTALL_COMMAND mkdir -p ${GZSTREAM_INSTALL_DIR}/lib/ && mkdir -p ${GZSTREAM_INSTALL_DIR}/include/ + && cp ${GZSTREAM_SOURCES_DIR}/src/extern_gzstream/libgzstream.a ${GZSTREAM_INSTALL_DIR}/lib + && cp -r ${GZSTREAM_SOURCES_DIR}/src/extern_gzstream/gzstream.h ${GZSTREAM_INSTALL_DIR}/include +) + +ADD_LIBRARY(gzstream STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET gzstream PROPERTY IMPORTED_LOCATION + "${GZSTREAM_INSTALL_DIR}/lib/libgzstream.a") + +include_directories(${GZSTREAM_INCLUDE_DIR}) +ADD_DEPENDENCIES(gzstream extern_gzstream) diff --git a/paddle/fluid/operators/reader/ctr_reader.h b/paddle/fluid/operators/reader/ctr_reader.h index 8a25993699213ae0710cff00bd469e5b1725a44f..1ef6e6d551fb155e7de12a370303a67992415b80 100644 --- a/paddle/fluid/operators/reader/ctr_reader.h +++ b/paddle/fluid/operators/reader/ctr_reader.h @@ -61,11 +61,12 @@ class CTRReader : public framework::FileReader { void Start() override { queue_->ReOpen(); - for (int i = 0; i < thread_num_; i++) { - read_threads_.emplace_back( - new std::thread(std::bind(&CTRReader::ReadThread, this, file_list_, - slots_, batch_size_, queue_))); - } + // for (int i = 0; i < thread_num_; i++) { + // read_threads_.emplace_back( + // new std::thread(std::bind(&CTRReader::ReadThread, this, + // file_list_, + // slots_, batch_size_, queue_))); + // } } private: