From d4822137f36c706ced937b6c4ee5028f77cbe181 Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Sat, 18 Apr 2020 15:21:06 +0800 Subject: [PATCH] fix segment bug on windows --- mindspore/ccsrc/CMakeLists.txt | 20 +++++------------ mindspore/ccsrc/dataset/CMakeLists.txt | 7 ++++-- .../dataset/engine/datasetops/dataset_op.cc | 4 ++++ .../ccsrc/dataset/engine/datasetops/map_op.cc | 22 +++++++++++++++++++ .../ccsrc/dataset/engine/datasetops/map_op.h | 4 ++++ mindspore/ccsrc/mindrecord/CMakeLists.txt | 2 +- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index e22796916..8c33b9051 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -156,7 +156,7 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ) if (ENABLE_CPU) list(REMOVE_ITEM MINDSPORE_SRC_LIST "device/gpu/distribution/collective_init.cc") - if (WIN32) + if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") list(REMOVE_ITEM MINDSPORE_SRC_LIST "kernel/kernel_query.cc") endif() endif() @@ -337,19 +337,19 @@ if (ENABLE_D) target_link_libraries(mindspore mindspore::protobuf) endif() -if (WIN32) +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_link_libraries(mindspore ${PYTHON_LIBRARIES} mindspore_gvar) endif() # set c_expression building -if (WIN32) -set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST} +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(PYTHON_MODULE_SOURCE ${MS_GVAR_SRC_LIST} pipeline/init.cc kernel/oplib/oplib.cc ${MINDSPORE_SRC_LIST} ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} ${GPU_NCCL_LIST} ${MS_HCCL_SRC_LIST} ${MS_PREDICT_SRC_LIST} ${CPU_SRC_LIST} ${MEM_REUSE_SRC_LIST} ${GPU_KERNEL_SRC_LIST}) else() -set(PYTHON_MODULE_SOURCE + set(PYTHON_MODULE_SOURCE pipeline/init.cc kernel/oplib/oplib.cc ${MS_STEPS_SRC_LIST} ${MS_CCE_SRC_LIST} ${MS_AICPU_SRC_LIST} ${MS_TASKINFO_LIST} ${MS_RT_SRC_LIST} @@ -426,13 +426,5 @@ endif() if(ENABLE_MINDDATA) add_subdirectory(mindrecord) - if (WIN32) - set(_md_tmp_CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) - set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O0 -Wl,--allow-shlib-undefined -DHALF_ENABLE_CPP11_USER_LITERALS=0") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -D_FORTIFY_SOURCE=2") - add_subdirectory(dataset) - set(CMAKE_CXX_FLAGS_RELEASE ${_md_tmp_CMAKE_CXX_FLAGS_RELEASE}) - else() - add_subdirectory(dataset) - endif() + add_subdirectory(dataset) endif() diff --git a/mindspore/ccsrc/dataset/CMakeLists.txt b/mindspore/ccsrc/dataset/CMakeLists.txt index b3ac34de7..0bc4065ac 100644 --- a/mindspore/ccsrc/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/CMakeLists.txt @@ -12,6 +12,9 @@ endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes") +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--image-base -Wl,0x10000000") +endif() ############################# Options ################################ if (ENABLE_GPUQUE) add_definitions(-D ENABLE_GPUQUE) @@ -80,7 +83,7 @@ set_target_properties(_c_dataengine PROPERTIES ###################################################################### ################# Link with external libraries ######################## -if (WIN32) +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_link_libraries(_c_dataengine PRIVATE mindspore) target_link_libraries(_c_dataengine PRIVATE mindspore::pybind11_module ${PYTHON_LIBRARIES} mindspore::protobuf ${SECUREC_LIBRARY}) else() @@ -101,7 +104,7 @@ if (ENABLE_TDTQUE) endif () add_dependencies(_c_dataengine _c_mindrecord) -if (WIN32) +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(MINDRECORD_LINK_OBJECT ${CMAKE_BINARY_DIR}/mindspore/ccsrc/mindrecord/CMakeFiles/_c_mindrecord.dir/objects.a) target_link_libraries(_c_dataengine PRIVATE _c_mindrecord ${MINDRECORD_LINK_OBJECT} mindspore::sqlite) else() diff --git a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc index 7edf1dd28..5e3ea3dc4 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc @@ -109,11 +109,15 @@ void DatasetOp::Print(std::ostream &out, bool show_all) const { // Gets the next buffer from the given child Status DatasetOp::GetNextBuffer(std::unique_ptr *p_buffer, int32_t worker_id, bool retry_if_eoe) { +#if defined(_WIN32) || defined(_WIN64) + RETURN_IF_NOT_OK(out_connector_->PopWithRetry(static_cast(worker_id), p_buffer, retry_if_eoe)); +#else std::unique_ptr next_buff; // pop is a blocked call and will throw an interruption if the whole group shuts down. RETURN_IF_NOT_OK(out_connector_->PopWithRetry(static_cast(worker_id), &next_buff, retry_if_eoe)); *p_buffer = std::move(next_buff); +#endif return Status::OK(); } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index b6d603bac..3f8d70b60 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -65,6 +65,9 @@ MapOp::MapOp(const std::vector &in_col_names, const std::vectorGetNextBuffer(&buff, 0)); is_eof = buff->eof(); RETURN_IF_NOT_OK(local_queues_[que_id]->Add(std::move(buff))); +#if defined(_WIN32) || defined(_WIN64) + if (is_eof) { + eof_worker_id_ = que_id; + for (int32_t id = 0; id < num_workers_; id++) { + if (id != eof_worker_id_) { + auto eof_buffer = std::make_unique(0, DataBuffer::kDeBFlagEOF); + RETURN_IF_NOT_OK(local_queues_[id]->Add(std::move(eof_buffer))); + } + } + } +#endif que_id = (que_id + 1) % num_workers_; } } @@ -159,6 +173,14 @@ Status MapOp::WorkerEntry(int32_t worker_id) { continue; } else if (in_buffer->eof()) { // Calling base class EofReceived to forward eof buffer. +#if defined(_WIN32) || defined(_Win64) + if (perf_mode_) { + if (eof_worker_id_ == worker_id) { + RETURN_IF_NOT_OK(EofReceived(worker_id)); + } + break; + } +#endif RETURN_IF_NOT_OK(EofReceived(worker_id)); break; } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h index 4c9d27f9c..5e16bc3fe 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -193,6 +193,10 @@ class MapOp : public ParallelOp { // cause additional blocking because pop calls to Connector from the threads are synchronized to enforce the order. bool perf_mode_; +#if defined(_WIN32) || defined(_WIN64) + // EOF worker id is only work on Performance mode, to record the worker id of queue which gets EOF + int32_t eof_worker_id_; +#endif // Private function for worker/thread to loop continuously. It comprises the main // logic of MapOp: getting the data from previous Op, validating user specified column names, // applying a list of TensorOps to each of the data, process the results and then diff --git a/mindspore/ccsrc/mindrecord/CMakeLists.txt b/mindspore/ccsrc/mindrecord/CMakeLists.txt index a2b8897b1..fdd648a50 100644 --- a/mindspore/ccsrc/mindrecord/CMakeLists.txt +++ b/mindspore/ccsrc/mindrecord/CMakeLists.txt @@ -26,7 +26,7 @@ set_target_properties(_c_mindrecord PROPERTIES ) # add link library -if (WIN32) +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite mindspore mindspore::protobuf) else() target_link_libraries(_c_mindrecord PRIVATE mindspore::sqlite ${PYTHON_LIB} ${SECUREC_LIBRARY} mindspore mindspore_gvar mindspore::protobuf) -- GitLab