Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
magicwindyyd
mindspore
提交
525d063a
M
mindspore
项目概览
magicwindyyd
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
525d063a
编写于
6月 05, 2020
作者:
Y
Yang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[Dataset] Fix codedex.
上级
5812c46e
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
12 addition
and
12 deletion
+12
-12
mindspore/ccsrc/mindrecord/include/shard_column.h
mindspore/ccsrc/mindrecord/include/shard_column.h
+3
-3
mindspore/ccsrc/mindrecord/meta/shard_column.cc
mindspore/ccsrc/mindrecord/meta/shard_column.cc
+9
-9
未找到文件。
mindspore/ccsrc/mindrecord/include/shard_column.h
浏览文件 @
525d063a
...
...
@@ -67,7 +67,7 @@ class ShardColumn {
/// \brief get column value by column name
MSRStatus
GetColumnValueByName
(
const
std
::
string
&
column_name
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
const
json
&
columns_json
,
const
unsigned
char
**
data
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
n_bytes
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
const
n_bytes
,
ColumnDataType
*
column_data_type
,
uint64_t
*
column_data_type_size
,
std
::
vector
<
int64_t
>
*
column_shape
);
...
...
@@ -88,7 +88,7 @@ class ShardColumn {
/// \brief get column value from blob
MSRStatus
GetColumnFromBlob
(
const
std
::
string
&
column_name
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
const
unsigned
char
**
data
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
n_bytes
);
uint64_t
*
const
n_bytes
);
std
::
pair
<
MSRStatus
,
ColumnCategory
>
GetColumnTypeByName
(
const
std
::
string
&
column_name
,
ColumnDataType
*
column_data_type
,
uint64_t
*
column_data_type_size
,
...
...
@@ -119,7 +119,7 @@ class ShardColumn {
/// \brief uncompress integer array column
template
<
typename
T
>
static
MSRStatus
UncompressInt
(
const
uint64_t
&
column_id
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
static
MSRStatus
UncompressInt
(
const
uint64_t
&
column_id
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
const
data_ptr
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
uint64_t
*
num_bytes
,
uint64_t
shift_idx
);
/// \brief convert big-endian bytes to unsigned int
...
...
mindspore/ccsrc/mindrecord/meta/shard_column.cc
浏览文件 @
525d063a
...
...
@@ -87,7 +87,7 @@ std::pair<MSRStatus, ColumnCategory> ShardColumn::GetColumnTypeByName(const std:
MSRStatus
ShardColumn
::
GetColumnValueByName
(
const
std
::
string
&
column_name
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
const
json
&
columns_json
,
const
unsigned
char
**
data
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
n_bytes
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
const
n_bytes
,
ColumnDataType
*
column_data_type
,
uint64_t
*
column_data_type_size
,
std
::
vector
<
int64_t
>
*
column_shape
)
{
// Skip if column not found
...
...
@@ -250,7 +250,7 @@ MSRStatus ShardColumn::GetInt(std::unique_ptr<unsigned char[]> *data_ptr, const
MSRStatus
ShardColumn
::
GetColumnFromBlob
(
const
std
::
string
&
column_name
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
const
unsigned
char
**
data
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
uint64_t
*
n_bytes
)
{
uint64_t
*
const
n_bytes
)
{
uint64_t
offset_address
=
0
;
auto
column_id
=
column_name_id_
[
column_name
];
if
(
GetColumnAddressInBlock
(
column_id
,
columns_blob
,
n_bytes
,
&
offset_address
)
==
FAILED
)
{
...
...
@@ -357,7 +357,7 @@ vector<uint8_t> ShardColumn::CompressInt(const vector<uint8_t> &src_bytes, const
// Update date type in bit map
dst_bytes
[
i
/
kNumDataOfByte
+
kBytesOfColumnLen
]
|=
(
dst_int_type
<<
(
kDataTypeBits
*
(
kNumDataOfByte
-
kUnsignedOne
-
(
i
%
kNumDataOfByte
))));
(
static_cast
<
uint8_t
>
(
dst_int_type
)
<<
(
kDataTypeBits
*
(
kNumDataOfByte
-
kUnsignedOne
-
(
i
%
kNumDataOfByte
))));
}
// Resize destination blob
dst_bytes
.
resize
(
i_dst
);
...
...
@@ -385,7 +385,7 @@ MSRStatus ShardColumn::GetColumnAddressInBlock(const uint64_t &column_id, const
}
template
<
typename
T
>
MSRStatus
ShardColumn
::
UncompressInt
(
const
uint64_t
&
column_id
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
data_ptr
,
MSRStatus
ShardColumn
::
UncompressInt
(
const
uint64_t
&
column_id
,
std
::
unique_ptr
<
unsigned
char
[]
>
*
const
data_ptr
,
const
std
::
vector
<
uint8_t
>
&
columns_blob
,
uint64_t
*
num_bytes
,
uint64_t
shift_idx
)
{
auto
num_elements
=
BytesBigToUInt64
(
columns_blob
,
shift_idx
,
kInt32Type
);
...
...
@@ -406,7 +406,7 @@ MSRStatus ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr<
auto
data
=
reinterpret_cast
<
const
unsigned
char
*>
(
array_data
.
get
());
*
data_ptr
=
std
::
make_unique
<
unsigned
char
[]
>
(
*
num_bytes
);
memcpy
(
data_ptr
->
get
()
,
data
,
*
num_bytes
);
memcpy
_s
(
data_ptr
->
get
(),
*
num_bytes
,
data
,
*
num_bytes
);
return
SUCCESS
;
}
...
...
@@ -414,14 +414,14 @@ MSRStatus ShardColumn::UncompressInt(const uint64_t &column_id, std::unique_ptr<
uint64_t
ShardColumn
::
BytesBigToUInt64
(
const
std
::
vector
<
uint8_t
>
&
bytes_array
,
const
uint64_t
&
pos
,
const
IntegerType
&
i_type
)
{
uint64_t
result
=
0
;
for
(
uint64_t
i
=
0
;
i
<
(
kUnsignedOne
<<
i_type
);
i
++
)
{
for
(
uint64_t
i
=
0
;
i
<
(
kUnsignedOne
<<
static_cast
<
uint8_t
>
(
i_type
)
);
i
++
)
{
result
=
(
result
<<
kBitsOfByte
)
+
bytes_array
[
pos
+
i
];
}
return
result
;
}
std
::
vector
<
uint8_t
>
ShardColumn
::
UIntToBytesBig
(
uint64_t
value
,
const
IntegerType
&
i_type
)
{
uint64_t
n_bytes
=
kUnsignedOne
<<
i_type
;
uint64_t
n_bytes
=
kUnsignedOne
<<
static_cast
<
uint8_t
>
(
i_type
)
;
std
::
vector
<
uint8_t
>
result
(
n_bytes
,
0
);
for
(
uint64_t
i
=
0
;
i
<
n_bytes
;
i
++
)
{
result
[
n_bytes
-
1
-
i
]
=
value
&
std
::
numeric_limits
<
uint8_t
>::
max
();
...
...
@@ -431,7 +431,7 @@ std::vector<uint8_t> ShardColumn::UIntToBytesBig(uint64_t value, const IntegerTy
}
std
::
vector
<
uint8_t
>
ShardColumn
::
UIntToBytesLittle
(
uint64_t
value
,
const
IntegerType
&
i_type
)
{
uint64_t
n_bytes
=
kUnsignedOne
<<
i_type
;
uint64_t
n_bytes
=
kUnsignedOne
<<
static_cast
<
uint8_t
>
(
i_type
)
;
std
::
vector
<
uint8_t
>
result
(
n_bytes
,
0
);
for
(
uint64_t
i
=
0
;
i
<
n_bytes
;
i
++
)
{
result
[
i
]
=
value
&
std
::
numeric_limits
<
uint8_t
>::
max
();
...
...
@@ -443,7 +443,7 @@ std::vector<uint8_t> ShardColumn::UIntToBytesLittle(uint64_t value, const Intege
int64_t
ShardColumn
::
BytesLittleToMinIntType
(
const
std
::
vector
<
uint8_t
>
&
bytes_array
,
const
uint64_t
&
pos
,
const
IntegerType
&
src_i_type
,
IntegerType
*
dst_i_type
)
{
uint64_t
u_temp
=
0
;
for
(
uint64_t
i
=
0
;
i
<
(
kUnsignedOne
<<
s
rc_i_type
);
i
++
)
{
for
(
uint64_t
i
=
0
;
i
<
(
kUnsignedOne
<<
s
tatic_cast
<
uint8_t
>
(
src_i_type
)
);
i
++
)
{
u_temp
=
(
u_temp
<<
kBitsOfByte
)
+
bytes_array
[
pos
+
(
kUnsignedOne
<<
src_i_type
)
-
kUnsignedOne
-
i
];
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录