Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
正统之独孤求败
mindspore
提交
20d1b644
M
mindspore
项目概览
正统之独孤求败
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
20d1b644
编写于
4月 08, 2020
作者:
J
jonyguo
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix: error info is not exactly when column list invalid
上级
606310d9
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
271 addition
and
53 deletion
+271
-53
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
...e/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
+2
-1
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h
...re/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h
+1
-0
mindspore/ccsrc/mindrecord/common/shard_error.cc
mindspore/ccsrc/mindrecord/common/shard_error.cc
+178
-0
mindspore/ccsrc/mindrecord/include/shard_error.h
mindspore/ccsrc/mindrecord/include/shard_error.h
+52
-50
mindspore/ccsrc/mindrecord/io/shard_reader.cc
mindspore/ccsrc/mindrecord/io/shard_reader.cc
+1
-1
tests/ut/cpp/dataset/mind_record_op_test.cc
tests/ut/cpp/dataset/mind_record_op_test.cc
+36
-0
tests/ut/cpp/mindrecord/ut_shard_reader_test.cc
tests/ut/cpp/mindrecord/ut_shard_reader_test.cc
+1
-1
未找到文件。
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
浏览文件 @
20d1b644
...
...
@@ -103,7 +103,8 @@ Status MindRecordOp::Init() {
shard_reader_
=
mindspore
::
make_unique
<
ShardReader
>
();
auto
rc
=
shard_reader_
->
Open
(
dataset_file_
,
num_mind_record_workers_
,
columns_to_load_
,
operators_
,
block_reader_
);
CHECK_FAIL_RETURN_UNEXPECTED
(
rc
!=
MSRStatus
::
FAILED
,
"MindRecordOp init failed."
);
CHECK_FAIL_RETURN_UNEXPECTED
(
rc
!=
MSRStatus
::
FAILED
,
"MindRecordOp init failed. Error message: "
+
ErrnoToMessage
(
rc
));
data_schema_
=
mindspore
::
make_unique
<
DataSchema
>
();
...
...
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h
浏览文件 @
20d1b644
...
...
@@ -32,6 +32,7 @@
#include "dataset/engine/datasetops/source/io_block.h"
#include "dataset/util/queue.h"
#include "dataset/util/status.h"
#include "mindrecord/include/shard_error.h"
#include "mindrecord/include/shard_reader.h"
#include "mindrecord/include/common/shard_utils.h"
#include "dataset/util/wait_post.h"
...
...
mindspore/ccsrc/mindrecord/common/shard_error.cc
0 → 100644
浏览文件 @
20d1b644
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "mindrecord/include/shard_error.h"
namespace
mindspore
{
namespace
mindrecord
{
std
::
string
ErrnoToMessage
(
MSRStatus
status
)
{
switch
(
status
)
{
case
FAILED
:
return
"operator failed"
;
break
;
case
SUCCESS
:
return
"operator success"
;
break
;
case
OPEN_FILE_FAILED
:
return
"open file failed"
;
break
;
case
CLOSE_FILE_FAILED
:
return
"close file failed"
;
break
;
case
WRITE_METADATA_FAILED
:
return
"write metadata failed"
;
break
;
case
WRITE_RAWDATA_FAILED
:
return
"write rawdata failed"
;
break
;
case
GET_SCHEMA_FAILED
:
return
"get schema failed"
;
break
;
case
ILLEGAL_RAWDATA
:
return
"illegal raw data"
;
break
;
case
PYTHON_TO_JSON_FAILED
:
return
"pybind: python object to json failed"
;
break
;
case
DIR_CREATE_FAILED
:
return
"directory create failed"
;
break
;
case
OPEN_DIR_FAILED
:
return
"open directory failed"
;
break
;
case
INVALID_STATISTICS
:
return
"invalid statistics object"
;
break
;
case
OPEN_DATABASE_FAILED
:
return
"open database failed"
;
break
;
case
CLOSE_DATABASE_FAILED
:
return
"close database failed"
;
break
;
case
DATABASE_OPERATE_FAILED
:
return
"database operate failed"
;
break
;
case
BUILD_SCHEMA_FAILED
:
return
"build schema failed"
;
break
;
case
DIVISOR_IS_ILLEGAL
:
return
"divisor is illegal"
;
break
;
case
INVALID_FILE_PATH
:
return
"file path is invalid"
;
break
;
case
SECURE_FUNC_FAILED
:
return
"secure function failed"
;
break
;
case
ALLOCATE_MEM_FAILED
:
return
"allocate memory failed"
;
break
;
case
ILLEGAL_FIELD_NAME
:
return
"illegal field name"
;
break
;
case
ILLEGAL_FIELD_TYPE
:
return
"illegal field type"
;
break
;
case
SET_METADATA_FAILED
:
return
"set metadata failed"
;
break
;
case
ILLEGAL_SCHEMA_DEFINITION
:
return
"illegal schema definition"
;
break
;
case
ILLEGAL_COLUMN_LIST
:
return
"illegal column list"
;
break
;
case
SQL_ERROR
:
return
"sql error"
;
break
;
case
ILLEGAL_SHARD_COUNT
:
return
"illegal shard count"
;
break
;
case
ILLEGAL_SCHEMA_COUNT
:
return
"illegal schema count"
;
break
;
case
VERSION_ERROR
:
return
"data version is not matched"
;
break
;
case
ADD_SCHEMA_FAILED
:
return
"add schema failed"
;
break
;
case
ILLEGAL_Header_SIZE
:
return
"illegal header size"
;
break
;
case
ILLEGAL_Page_SIZE
:
return
"illegal page size"
;
break
;
case
ILLEGAL_SIZE_VALUE
:
return
"illegal size value"
;
break
;
case
INDEX_FIELD_ERROR
:
return
"add index fields failed"
;
break
;
case
GET_CANDIDATE_CATEGORYFIELDS_FAILED
:
return
"get candidate category fields failed"
;
break
;
case
GET_CATEGORY_INFO_FAILED
:
return
"get category information failed"
;
break
;
case
ILLEGAL_CATEGORY_ID
:
return
"illegal category id"
;
break
;
case
ILLEGAL_ROWNUMBER_OF_PAGE
:
return
"illegal row number of page"
;
break
;
case
ILLEGAL_SCHEMA_ID
:
return
"illegal schema id"
;
break
;
case
DESERIALIZE_SCHEMA_FAILED
:
return
"deserialize schema failed"
;
break
;
case
DESERIALIZE_STATISTICS_FAILED
:
return
"deserialize statistics failed"
;
break
;
case
ILLEGAL_DB_FILE
:
return
"illegal db file"
;
break
;
case
OVERWRITE_DB_FILE
:
return
"overwrite db file"
;
break
;
case
OVERWRITE_MINDRECORD_FILE
:
return
"overwrite mindrecord file"
;
break
;
case
ILLEGAL_MINDRECORD_FILE
:
return
"illegal mindrecord file"
;
break
;
case
PARSE_JSON_FAILED
:
return
"parse json failed"
;
break
;
case
ILLEGAL_PARAMETERS
:
return
"illegal parameters"
;
break
;
case
GET_PAGE_BY_GROUP_ID_FAILED
:
return
"get page by group id failed"
;
break
;
case
GET_SYSTEM_STATE_FAILED
:
return
"get system state failed"
;
break
;
case
IO_FAILED
:
return
"io operate failed"
;
break
;
default:
return
"invalid error no"
;
}
}
}
// namespace mindrecord
}
// namespace mindspore
mindspore/ccsrc/mindrecord/include/shard_error.h
浏览文件 @
20d1b644
...
...
@@ -18,63 +18,65 @@
#define MINDRECORD_INCLUDE_SHARD_ERROR_H_
#include <map>
#include
"utils/error_code.h"
#include
<string>
namespace
mindspore
{
namespace
mindrecord
{
DE_ERRORNO_MINDRECORD
(
OPEN_FILE_FAILED
,
0
,
"open file failed"
);
DE_ERRORNO_MINDRECORD
(
CLOSE_FILE_FAILED
,
1
,
"close file failed"
);
DE_ERRORNO_MINDRECORD
(
WRITE_METADATA_FAILED
,
2
,
"write metadata failed"
);
DE_ERRORNO_MINDRECORD
(
WRITE_RAWDATA_FAILED
,
3
,
"write rawdata failed"
);
DE_ERRORNO_MINDRECORD
(
GET_SCHEMA_FAILED
,
4
,
"get schema failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_RAWDATA
,
5
,
"illegal raw data"
);
DE_ERRORNO_MINDRECORD
(
PYTHON_TO_JSON_FAILED
,
6
,
"pybind: python object to json failed"
);
DE_ERRORNO_MINDRECORD
(
DIR_CREATE_FAILED
,
7
,
"directory create failed"
);
DE_ERRORNO_MINDRECORD
(
OPEN_DIR_FAILED
,
8
,
"open directory failed"
);
DE_ERRORNO_MINDRECORD
(
INVALID_STATISTICS
,
9
,
"invalid statistics object"
);
DE_ERRORNO_MINDRECORD
(
OPEN_DATABASE_FAILED
,
10
,
"open database failed"
);
DE_ERRORNO_MINDRECORD
(
CLOSE_DATABASE_FAILED
,
11
,
"close database failed"
);
DE_ERRORNO_MINDRECORD
(
DATABASE_OPERATE_FAILED
,
12
,
"database operate failed"
);
DE_ERRORNO_MINDRECORD
(
BUILD_SCHEMA_FAILED
,
13
,
"build schema failed"
);
DE_ERRORNO_MINDRECORD
(
DIVISOR_IS_ILLEGAL
,
14
,
"divisor is illegal"
);
DE_ERRORNO_MINDRECORD
(
INVALID_FILE_PATH
,
15
,
"file path is invalid"
);
DE_ERRORNO_MINDRECORD
(
SECURE_FUNC_FAILED
,
16
,
"secure function failed"
);
DE_ERRORNO_MINDRECORD
(
ALLOCATE_MEM_FAILED
,
17
,
"allocate memory failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_FIELD_NAME
,
18
,
"illegal field name"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_FIELD_TYPE
,
19
,
"illegal field type"
);
DE_ERRORNO_MINDRECORD
(
SET_METADATA_FAILED
,
20
,
"set metadata failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_SCHEMA_DEFINITION
,
21
,
"illegal schema definition"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_COLUMN_LIST
,
22
,
"illegal column list"
);
DE_ERRORNO_MINDRECORD
(
SQL_ERROR
,
23
,
"sql error"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_SHARD_COUNT
,
24
,
"illegal shard count"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_SCHEMA_COUNT
,
25
,
"illegal schema count"
);
DE_ERRORNO_MINDRECORD
(
VERSION_ERROR
,
26
,
"data version is not matched"
);
DE_ERRORNO_MINDRECORD
(
ADD_SCHEMA_FAILED
,
27
,
"add schema failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_Header_SIZE
,
28
,
"illegal header size"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_Page_SIZE
,
29
,
"illegal page size"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_SIZE_VALUE
,
30
,
"illegal size value"
);
DE_ERRORNO_MINDRECORD
(
INDEX_FIELD_FAILED
,
31
,
"add index fields failed"
);
DE_ERRORNO_MINDRECORD
(
GET_CANDIDATE_CATEGORYFIELDS_FAILED
,
32
,
"get candidate categoryFields failed"
);
DE_ERRORNO_MINDRECORD
(
GET_CATEGORY_INFO
,
33
,
"get category information failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_CATEGORY_ID
,
34
,
"illegal category id"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_ROWNUMBER_OF_PAGE
,
35
,
"illegal row number of page"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_SCHEMA_ID
,
36
,
"illegal schema id"
);
DE_ERRORNO_MINDRECORD
(
DESERIALIZE_SCHEMA_FAILED
,
37
,
"deserialize schema failed"
);
DE_ERRORNO_MINDRECORD
(
DESERIALIZE_STATISTICS_FAILED
,
38
,
"deserialize statistics failed"
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_DB_FILE
,
39
,
"illegal db file."
);
DE_ERRORNO_MINDRECORD
(
OVERWRITE_DB_FILE
,
40
,
"overwrite db file."
);
DE_ERRORNO_MINDRECORD
(
OVERWRITE_MINDRECORD_FILE
,
41
,
"overwrite mindrecord file."
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_MINDRECORD_FILE
,
42
,
"illegal mindrecord file."
);
DE_ERRORNO_MINDRECORD
(
PARSE_JSON_FAILED
,
43
,
"parse json failed."
);
DE_ERRORNO_MINDRECORD
(
ILLEGAL_PARAMETERS
,
44
,
"illegal parameters."
);
DE_ERRORNO_MINDRECORD
(
GET_PAGE_BY_GROUP_ID_FAILED
,
46
,
"get page by group id failed."
);
DE_ERRORNO_MINDRECORD
(
GET_SYSTEM_STATE_FAILED
,
47
,
"get system state failed."
);
DE_ERRORNO_MINDRECORD
(
IO_FAILED
,
48
,
"io operate failed."
);
enum
MSRStatus
{
SUCCESS
=
0
,
FAILED
=
1
,
OPEN_FILE_FAILED
,
CLOSE_FILE_FAILED
,
WRITE_METADATA_FAILED
,
WRITE_RAWDATA_FAILED
,
GET_SCHEMA_FAILED
,
ILLEGAL_RAWDATA
,
PYTHON_TO_JSON_FAILED
,
DIR_CREATE_FAILED
,
OPEN_DIR_FAILED
,
INVALID_STATISTICS
,
OPEN_DATABASE_FAILED
,
CLOSE_DATABASE_FAILED
,
DATABASE_OPERATE_FAILED
,
BUILD_SCHEMA_FAILED
,
DIVISOR_IS_ILLEGAL
,
INVALID_FILE_PATH
,
SECURE_FUNC_FAILED
,
ALLOCATE_MEM_FAILED
,
ILLEGAL_FIELD_NAME
,
ILLEGAL_FIELD_TYPE
,
SET_METADATA_FAILED
,
ILLEGAL_SCHEMA_DEFINITION
,
ILLEGAL_COLUMN_LIST
,
SQL_ERROR
,
ILLEGAL_SHARD_COUNT
,
ILLEGAL_SCHEMA_COUNT
,
VERSION_ERROR
,
ADD_SCHEMA_FAILED
,
ILLEGAL_Header_SIZE
,
ILLEGAL_Page_SIZE
,
ILLEGAL_SIZE_VALUE
,
INDEX_FIELD_ERROR
,
GET_CANDIDATE_CATEGORYFIELDS_FAILED
,
GET_CATEGORY_INFO_FAILED
,
ILLEGAL_CATEGORY_ID
,
ILLEGAL_ROWNUMBER_OF_PAGE
,
ILLEGAL_SCHEMA_ID
,
DESERIALIZE_SCHEMA_FAILED
,
DESERIALIZE_STATISTICS_FAILED
,
ILLEGAL_DB_FILE
,
OVERWRITE_DB_FILE
,
OVERWRITE_MINDRECORD_FILE
,
ILLEGAL_MINDRECORD_FILE
,
PARSE_JSON_FAILED
,
ILLEGAL_PARAMETERS
,
GET_PAGE_BY_GROUP_ID_FAILED
,
GET_SYSTEM_STATE_FAILED
,
IO_FAILED
};
// convert error no to string message
std
::
string
ErrnoToMessage
(
MSRStatus
status
);
}
// namespace mindrecord
}
// namespace mindspore
...
...
mindspore/ccsrc/mindrecord/io/shard_reader.cc
浏览文件 @
20d1b644
...
...
@@ -676,7 +676,7 @@ MSRStatus ShardReader::Open(const std::string &file_path, int n_consumer,
if
(
CheckColumnList
(
selected_columns_
)
==
FAILED
)
{
MS_LOG
(
ERROR
)
<<
"Illegal column list"
;
return
FAILED
;
return
ILLEGAL_COLUMN_LIST
;
}
// Initialize argument
...
...
tests/ut/cpp/dataset/mind_record_op_test.cc
浏览文件 @
20d1b644
...
...
@@ -21,6 +21,7 @@
#include "common/utils.h"
#include "gtest/gtest.h"
#include "mindrecord/include/shard_category.h"
#include "mindrecord/include/shard_error.h"
#include "mindrecord/include/shard_sample.h"
#include "mindrecord/include/shard_shuffle.h"
#include "utils/log_adapter.h"
...
...
@@ -479,3 +480,38 @@ TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) {
row_count
++
;
}
}
TEST_F
(
MindDataTestMindRecordOp
,
TestMindRecordInvalidColumnList
)
{
// single MindRecord op and nothing else
//
// MindRecordOp
MS_LOG
(
INFO
)
<<
"UT test TestMindRecordInvalidColumnList"
;
Status
rc
;
// Start with an empty execution tree
auto
my_tree
=
std
::
make_shared
<
ExecutionTree
>
();
// Test info:
// Dataset from testDataset1 has 10 rows, 2 columns.
// RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row
// only. 2 workers.
// Test a column selection instead of all columns as well.
std
::
vector
<
std
::
string
>
column_list
;
std
::
string
label_col_name
(
"file_name_2"
);
column_list
.
push_back
(
label_col_name
);
label_col_name
=
"label"
;
column_list
.
push_back
(
label_col_name
);
std
::
shared_ptr
<
MindRecordOp
>
my_mindrecord_op
;
MindRecordOp
::
Builder
builder
;
builder
.
SetDatasetFile
(
mindrecord_root_path_
+
"/testMindDataSet/testImageNetData/imagenet.mindrecord0"
)
.
SetRowsPerBuffer
(
3
)
.
SetNumMindRecordWorkers
(
4
)
.
SetColumnsToLoad
(
column_list
);
rc
=
builder
.
Build
(
&
my_mindrecord_op
);
ASSERT_TRUE
(
rc
.
IsError
());
ASSERT_TRUE
(
rc
.
ToString
().
find_first_of
(
"illegal column list"
)
!=
std
::
string
::
npos
);
}
tests/ut/cpp/mindrecord/ut_shard_reader_test.cc
浏览文件 @
20d1b644
...
...
@@ -155,7 +155,7 @@ TEST_F(TestShardReader, TestShardReaderColumnNotInSchema) {
auto
column_list
=
std
::
vector
<
std
::
string
>
{
"file_namex"
};
ShardReader
dataset
;
MSRStatus
ret
=
dataset
.
Open
(
file_name
,
4
,
column_list
);
ASSERT_EQ
(
ret
,
FAILED
);
ASSERT_EQ
(
ret
,
ILLEGAL_COLUMN_LIST
);
}
TEST_F
(
TestShardReader
,
TestShardVersion
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录