Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
正统之独孤求败
mindspore
提交
332398d2
M
mindspore
项目概览
正统之独孤求败
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
332398d2
编写于
4月 28, 2020
作者:
J
Jesse Lee
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Check return code from WaitPost::Register
上级
7180bafc
变更
17
隐藏空白更改
内联
并排
Showing
17 changed file
with
17 addition
and
105 deletion
+17
-105
mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
...spore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc
...re/ccsrc/dataset/engine/datasetops/source/generator_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
...ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
...ore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
...e/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc
...re/ccsrc/dataset/engine/datasetops/source/text_file_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
...re/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
+1
-1
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
+1
-1
mindspore/ccsrc/dataset/util/CMakeLists.txt
mindspore/ccsrc/dataset/util/CMakeLists.txt
+0
-1
mindspore/ccsrc/dataset/util/semaphore.cc
mindspore/ccsrc/dataset/util/semaphore.cc
+0
-38
mindspore/ccsrc/dataset/util/semaphore.h
mindspore/ccsrc/dataset/util/semaphore.h
+0
-49
mindspore/ccsrc/dataset/util/task_manager.cc
mindspore/ccsrc/dataset/util/task_manager.cc
+1
-1
mindspore/ccsrc/dataset/util/wait_post.cc
mindspore/ccsrc/dataset/util/wait_post.cc
+1
-1
mindspore/ccsrc/dataset/util/wait_post.h
mindspore/ccsrc/dataset/util/wait_post.h
+1
-1
tests/ut/cpp/dataset/interrupt_test.cc
tests/ut/cpp/dataset/interrupt_test.cc
+4
-4
未找到文件。
mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
浏览文件 @
332398d2
...
...
@@ -94,7 +94,7 @@ Status CelebAOp::LaunchThreadsAndInitOp() {
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
RETURN_IF_NOT_OK
(
attr_info_queue_
->
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
RETURN_IF_NOT_OK
(
tree_
->
AllTasks
()
->
CreateAsyncTask
(
"Walking attr file"
,
std
::
bind
(
&
CelebAOp
::
ParseAttrFile
,
this
)));
RETURN_IF_NOT_OK
(
tree_
->
LaunchWorkers
(
num_workers_
,
std
::
bind
(
&
CelebAOp
::
WorkerEntry
,
this
,
std
::
placeholders
::
_1
)));
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
浏览文件 @
332398d2
...
...
@@ -149,7 +149,7 @@ Status CifarOp::LaunchThreadsAndInitOp() {
RETURN_STATUS_UNEXPECTED
(
"tree_ not set"
);
}
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
RETURN_IF_NOT_OK
(
tree_
->
AllTasks
()
->
CreateAsyncTask
(
"Get cifar data block"
,
std
::
bind
(
&
CifarOp
::
ReadCifarBlockDataAsync
,
this
)));
RETURN_IF_NOT_OK
(
tree_
->
LaunchWorkers
(
num_workers_
,
std
::
bind
(
&
CifarOp
::
WorkerEntry
,
this
,
std
::
placeholders
::
_1
)));
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/generator_op.cc
浏览文件 @
332398d2
...
...
@@ -168,7 +168,7 @@ Status GeneratorOp::FillBuffer(TensorQTable *tt) {
Status
GeneratorOp
::
operator
()()
{
// Handshake with TaskManager to synchronize thread creation
TaskManager
::
FindMe
()
->
Post
();
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
std
::
unique_ptr
<
DataBuffer
>
fetched_buffer
;
bool
eof
=
false
;
while
(
!
eof
)
{
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
浏览文件 @
332398d2
...
...
@@ -386,7 +386,7 @@ Status ImageFolderOp::LaunchThreadsAndInitOp() {
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
RETURN_IF_NOT_OK
(
folder_name_queue_
->
Register
(
tree_
->
AllTasks
()));
RETURN_IF_NOT_OK
(
image_name_queue_
->
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
// The following code launch 3 threads group
// 1) A thread that walks all folders and push the folder names to a util:Queue mFoldernameQueue.
// 2) Workers that pull foldername from mFoldernameQueue, walk it and return the sorted images to mImagenameQueue
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
浏览文件 @
332398d2
...
...
@@ -140,7 +140,7 @@ Status ManifestOp::LaunchThreadsAndInitOp() {
RETURN_STATUS_UNEXPECTED
(
"tree_ not set"
);
}
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
RETURN_IF_NOT_OK
(
tree_
->
LaunchWorkers
(
num_workers_
,
std
::
bind
(
&
ManifestOp
::
WorkerEntry
,
this
,
std
::
placeholders
::
_1
)));
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
浏览文件 @
332398d2
...
...
@@ -644,7 +644,7 @@ Status MindRecordOp::LaunchThreadAndInitOp() {
}
RETURN_IF_NOT_OK
(
io_blk_queues_
.
Register
(
tree_
->
AllTasks
()));
shard_reader_wait_post_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
shard_reader_wait_post_
.
Register
(
tree_
->
AllTasks
()
));
if
(
shard_reader_
->
Launch
(
!
block_reader_
)
==
MSRStatus
::
FAILED
)
{
RETURN_STATUS_UNEXPECTED
(
"MindRecordOp launch failed."
);
}
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
浏览文件 @
332398d2
...
...
@@ -395,7 +395,7 @@ Status MnistOp::LaunchThreadsAndInitOp() {
RETURN_STATUS_UNEXPECTED
(
"tree_ not set"
);
}
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
RETURN_IF_NOT_OK
(
tree_
->
LaunchWorkers
(
num_workers_
,
std
::
bind
(
&
MnistOp
::
WorkerEntry
,
this
,
std
::
placeholders
::
_1
)));
TaskManager
::
FindMe
()
->
Post
();
RETURN_IF_NOT_OK
(
this
->
WalkAllFiles
());
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc
浏览文件 @
332398d2
...
...
@@ -370,7 +370,7 @@ Status TextFileOp::operator()() {
// must be called after launching workers.
TaskManager
::
FindMe
()
->
Post
();
io_block_queue_wait_post_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
io_block_queue_wait_post_
.
Register
(
tree_
->
AllTasks
()
));
NotifyToFillIOBlockQueue
();
while
(
!
finished_reading_dataset_
)
{
int64_t
buffer_id
=
0
;
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
浏览文件 @
332398d2
...
...
@@ -222,7 +222,7 @@ Status TFReaderOp::operator()() {
// so workers have to be kept alive until the end of the program
TaskManager
::
FindMe
()
->
Post
();
io_block_queue_wait_post_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
io_block_queue_wait_post_
.
Register
(
tree_
->
AllTasks
()
));
NotifyToFillIOBlockQueue
();
while
(
!
finished_reading_dataset_
)
{
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
浏览文件 @
332398d2
...
...
@@ -231,7 +231,7 @@ Status VOCOp::LaunchThreadsAndInitOp() {
RETURN_STATUS_UNEXPECTED
(
"tree_ not set"
);
}
RETURN_IF_NOT_OK
(
io_block_queues_
.
Register
(
tree_
->
AllTasks
()));
wp_
.
Register
(
tree_
->
AllTasks
(
));
RETURN_IF_NOT_OK
(
wp_
.
Register
(
tree_
->
AllTasks
()
));
RETURN_IF_NOT_OK
(
tree_
->
LaunchWorkers
(
num_workers_
,
std
::
bind
(
&
VOCOp
::
WorkerEntry
,
this
,
std
::
placeholders
::
_1
)));
TaskManager
::
FindMe
()
->
Post
();
RETURN_IF_NOT_OK
(
this
->
ParseImageIds
());
...
...
mindspore/ccsrc/dataset/util/CMakeLists.txt
浏览文件 @
332398d2
...
...
@@ -3,7 +3,6 @@ add_library(utils OBJECT
circular_pool.cc
memory_pool.cc
cond_var.cc
semaphore.cc
intrp_service.cc
task.cc
task_manager.cc
...
...
mindspore/ccsrc/dataset/util/semaphore.cc
已删除
100644 → 0
浏览文件 @
7180bafc
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "dataset/util/semaphore.h"
#include "dataset/util/task_manager.h"
namespace
mindspore
{
namespace
dataset
{
Status
Semaphore
::
P
()
{
std
::
unique_lock
<
std
::
mutex
>
lck
(
mutex_
);
return
(
wait_cond_
.
Wait
(
&
lck
,
[
this
]()
{
return
value_
!=
0
;
}));
}
void
Semaphore
::
V
()
{
std
::
unique_lock
<
std
::
mutex
>
lck
(
mutex_
);
++
value_
;
wait_cond_
.
NotifyOne
();
}
void
Semaphore
::
Register
(
TaskGroup
*
vg
)
{
(
void
)
wait_cond_
.
Register
(
vg
->
GetIntrpService
());
}
Status
Semaphore
::
Deregister
()
{
return
(
wait_cond_
.
Deregister
());
}
void
Semaphore
::
ResetIntrpState
()
{
wait_cond_
.
ResetIntrpState
();
}
}
// namespace dataset
}
// namespace mindspore
mindspore/ccsrc/dataset/util/semaphore.h
已删除
100644 → 0
浏览文件 @
7180bafc
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef DATASET_UTIL_SEMAPHORE_H_
#define DATASET_UTIL_SEMAPHORE_H_
#include "dataset/util/cond_var.h"
namespace
mindspore
{
namespace
dataset
{
class
TaskGroup
;
class
Semaphore
{
public:
explicit
Semaphore
(
int
init
)
:
value_
(
init
)
{}
virtual
~
Semaphore
()
{}
Status
P
();
void
V
();
void
Register
(
TaskGroup
*
vg
);
Status
Deregister
();
void
ResetIntrpState
();
private:
int
value_
;
std
::
mutex
mutex_
;
CondVar
wait_cond_
;
};
}
// namespace dataset
}
// namespace mindspore
#endif // DATASET_UTIL_SEMAPHORE_H_
mindspore/ccsrc/dataset/util/task_manager.cc
浏览文件 @
332398d2
...
...
@@ -53,7 +53,7 @@ Status TaskManager::CreateAsyncTask(const std::string &my_name, const std::funct
LockGuard
lck
(
&
tg_lock_
);
this
->
grp_list_
.
insert
(
vg
);
}
(
*
task
)
->
wp_
.
Register
(
vg
);
RETURN_IF_NOT_OK
((
*
task
)
->
wp_
.
Register
(
vg
)
);
RETURN_IF_NOT_OK
((
*
task
)
->
Run
());
// Wait for the thread to initialize successfully.
RETURN_IF_NOT_OK
((
*
task
)
->
Wait
());
...
...
mindspore/ccsrc/dataset/util/wait_post.cc
浏览文件 @
332398d2
...
...
@@ -36,7 +36,7 @@ void WaitPost::Clear() {
value_
=
0
;
}
void
WaitPost
::
Register
(
TaskGroup
*
vg
)
{
(
void
)
wait_cond_
.
Register
(
vg
->
GetIntrpService
());
}
Status
WaitPost
::
Register
(
TaskGroup
*
vg
)
{
return
wait_cond_
.
Register
(
vg
->
GetIntrpService
());
}
void
WaitPost
::
ResetIntrpState
()
{
wait_cond_
.
ResetIntrpState
();
}
...
...
mindspore/ccsrc/dataset/util/wait_post.h
浏览文件 @
332398d2
...
...
@@ -36,7 +36,7 @@ class WaitPost {
void
Clear
();
void
Register
(
TaskGroup
*
vg
);
Status
Register
(
TaskGroup
*
vg
);
Status
Deregister
();
...
...
tests/ut/cpp/dataset/interrupt_test.cc
浏览文件 @
332398d2
...
...
@@ -20,7 +20,6 @@
#include "dataset/util/intrp_service.h"
#include "dataset/util/task_manager.h"
#include "dataset/util/queue.h"
#include "dataset/util/semaphore.h"
using
namespace
mindspore
::
dataset
;
using
mindspore
::
MsLogLevel
::
INFO
;
...
...
@@ -55,11 +54,12 @@ TEST_F(MindDataTestIntrpService, Test1) {
TEST_F
(
MindDataTestIntrpService
,
Test2
)
{
MS_LOG
(
INFO
)
<<
"Test Semaphore"
;
Status
rc
;
Semaphore
sem
(
0
);
sem
.
Register
(
&
vg_
);
WaitPost
wp
;
rc
=
wp
.
Register
(
&
vg_
);
EXPECT_TRUE
(
rc
.
IsOk
());
vg_
.
CreateAsyncTask
(
"Test1"
,
[
&
]()
->
Status
{
TaskManager
::
FindMe
()
->
Post
();
Status
rc
=
sem
.
P
();
Status
rc
=
wp
.
Wait
();
EXPECT_TRUE
(
rc
.
IsInterrupted
());
return
rc
;
});
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录