Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
MindSpore
mindinsight
提交
5b455143
M
mindinsight
项目概览
MindSpore
/
mindinsight
通知
7
Star
3
Fork
2
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindinsight
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
5b455143
编写于
4月 14, 2020
作者:
李
李鸿章
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
resolve the crc32 c/c++ extension
上级
c84d33d9
变更
14
显示空白变更内容
内联
并排
Showing
14 changed file
with
98 addition
and
120 deletion
+98
-120
mindinsight/datavisual/data_transform/ms_data_loader.py
mindinsight/datavisual/data_transform/ms_data_loader.py
+2
-2
mindinsight/datavisual/utils/crc32/__init__.pyi
mindinsight/datavisual/utils/crc32/__init__.pyi
+23
-0
mindinsight/datavisual/utils/crc32/base.h
mindinsight/datavisual/utils/crc32/base.h
+0
-57
mindinsight/datavisual/utils/crc32/crc32.cc
mindinsight/datavisual/utils/crc32/crc32.cc
+21
-22
mindinsight/datavisual/utils/crc32/crc32.h
mindinsight/datavisual/utils/crc32/crc32.h
+42
-14
mindinsight/lineagemgr/summary/event_writer.py
mindinsight/lineagemgr/summary/event_writer.py
+1
-7
mindinsight/lineagemgr/summary/lineage_summary_analyzer.py
mindinsight/lineagemgr/summary/lineage_summary_analyzer.py
+1
-2
tests/ut/datavisual/data_transform/test_data_manager.py
tests/ut/datavisual/data_transform/test_data_manager.py
+1
-2
tests/ut/datavisual/data_transform/test_ms_data_loader.py
tests/ut/datavisual/data_transform/test_ms_data_loader.py
+2
-4
tests/ut/datavisual/processors/test_graph_processor.py
tests/ut/datavisual/processors/test_graph_processor.py
+1
-2
tests/ut/datavisual/processors/test_images_processor.py
tests/ut/datavisual/processors/test_images_processor.py
+1
-2
tests/ut/datavisual/processors/test_scalars_processor.py
tests/ut/datavisual/processors/test_scalars_processor.py
+1
-2
tests/ut/datavisual/processors/test_train_task_manager.py
tests/ut/datavisual/processors/test_train_task_manager.py
+1
-2
tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py
tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py
+1
-2
未找到文件。
mindinsight/datavisual/data_transform/ms_data_loader.py
浏览文件 @
5b455143
...
@@ -184,7 +184,7 @@ class MSDataLoader:
...
@@ -184,7 +184,7 @@ class MSDataLoader:
logger
.
warning
(
"Check header size and crc, record truncated at offset %s, "
logger
.
warning
(
"Check header size and crc, record truncated at offset %s, "
"file_path=%s."
,
file_handler
.
offset
,
file_handler
.
file_path
)
"file_path=%s."
,
file_handler
.
offset
,
file_handler
.
file_path
)
return
None
return
None
if
crc32
.
GetValueFromStr
(
header_crc_str
)
!=
crc32
.
GetMaskCrc32cValue
(
header_str
,
HEADER_SIZE
):
if
not
crc32
.
CheckValueAgainstData
(
header_crc_str
,
header_str
,
HEADER_SIZE
):
raise
exceptions
.
CRCFailedError
()
raise
exceptions
.
CRCFailedError
()
# read the event body if integrity of header is verified
# read the event body if integrity of header is verified
...
@@ -202,7 +202,7 @@ class MSDataLoader:
...
@@ -202,7 +202,7 @@ class MSDataLoader:
logger
.
warning
(
"Check event crc, record truncated at offset %d, file_path: %s."
,
logger
.
warning
(
"Check event crc, record truncated at offset %d, file_path: %s."
,
file_handler
.
offset
,
file_handler
.
file_path
)
file_handler
.
offset
,
file_handler
.
file_path
)
return
None
return
None
if
crc32
.
GetValueFromStr
(
event_crc_str
)
!=
crc32
.
GetMaskCrc32cValue
(
event_str
,
event_len
):
if
not
crc32
.
CheckValueAgainstData
(
event_crc_str
,
event_str
,
event_len
):
raise
exceptions
.
CRCFailedError
()
raise
exceptions
.
CRCFailedError
()
return
event_str
return
event_str
...
...
mindinsight/datavisual/utils/crc32/__init__.pyi
0 → 100644
浏览文件 @
5b455143
# Copyright 2019 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""crc32 type stub module."""
def CheckValueAgainstData(crc_value: bytes, data: bytes, size: int) -> bool:
"""Check crc_value against new crc value from data to see if data is currupted."""
def GetMaskCrc32cValue(data: bytes, n: int) -> int:
"""Get masked crc value from data."""
mindinsight/datavisual/utils/crc32/base.h
已删除
100644 → 0
浏览文件 @
c84d33d9
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef DATAVISUAL_UTILS_CRC32_BASE_H_
#define DATAVISUAL_UTILS_CRC32_BASE_H_
#include <memory>
#include <string>
#include "securec/include/securec.h"
using
string
=
std
::
string
;
using
int8
=
int8_t
;
using
int16
=
int16_t
;
using
int32
=
int32_t
;
using
int64
=
int64_t
;
using
uint8
=
uint8_t
;
using
uint16
=
uint16_t
;
using
uint32
=
uint32_t
;
using
uint64
=
uint64_t
;
// check the null point, Only log it in if(): The value is null
#define EXCEPT_CHECK_NULL(value) \
do { \
if (value == nullptr) { \
break; \
} \
} while (0)
// implement common define function
// Get the 32 bits align value
inline
uint32
DecodeFixed32
(
const
char
*
ptr
)
{
uint32
result
=
0
;
if
(
EOK
!=
memcpy_s
(
&
result
,
sizeof
(
result
),
ptr
,
sizeof
(
result
)))
{
return
result
;
}
return
result
;
}
// Used to fetch a naturally-aligned 32-bit word in little endian byte-order
inline
uint32
LE_LOAD32
(
const
uint8_t
*
p
)
{
return
DecodeFixed32
(
reinterpret_cast
<
const
char
*>
(
p
));
}
#endif // DATAVISUAL_UTILS_CRC32_BASE_H_
mindinsight/datavisual/utils/crc32/crc32.cc
浏览文件 @
5b455143
...
@@ -17,9 +17,8 @@
...
@@ -17,9 +17,8 @@
#include "crc32/crc32.h"
#include "crc32/crc32.h"
#include <stdint.h>
#include <stdint.h>
const
unsigned
int
CRC_TABLE_SIZE
=
256
;
static
const
uint32
crc_table_o32
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o32
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0xF26B8303
,
0xE13B70F7
,
0x1350F3F4
,
0xC79A971F
,
0x35F1141C
,
0x26A1E7E8
,
0xD4CA64EB
,
0x8AD958CF
,
0x00000000
,
0xF26B8303
,
0xE13B70F7
,
0x1350F3F4
,
0xC79A971F
,
0x35F1141C
,
0x26A1E7E8
,
0xD4CA64EB
,
0x8AD958CF
,
0x78B2DBCC
,
0x6BE22838
,
0x9989AB3B
,
0x4D43CFD0
,
0xBF284CD3
,
0xAC78BF27
,
0x5E133C24
,
0x105EC76F
,
0xE235446C
,
0x78B2DBCC
,
0x6BE22838
,
0x9989AB3B
,
0x4D43CFD0
,
0xBF284CD3
,
0xAC78BF27
,
0x5E133C24
,
0x105EC76F
,
0xE235446C
,
0xF165B798
,
0x030E349B
,
0xD7C45070
,
0x25AFD373
,
0x36FF2087
,
0xC494A384
,
0x9A879FA0
,
0x68EC1CA3
,
0x7BBCEF57
,
0xF165B798
,
0x030E349B
,
0xD7C45070
,
0x25AFD373
,
0x36FF2087
,
0xC494A384
,
0x9A879FA0
,
0x68EC1CA3
,
0x7BBCEF57
,
...
@@ -50,7 +49,7 @@ static const uint32 crc_table_o32[CRC_TABLE_SIZE] = {
...
@@ -50,7 +49,7 @@ static const uint32 crc_table_o32[CRC_TABLE_SIZE] = {
0xE03E9C81
,
0x34F4F86A
,
0xC69F7B69
,
0xD5CF889D
,
0x27A40B9E
,
0x79B737BA
,
0x8BDCB4B9
,
0x988C474D
,
0x6AE7C44E
,
0xE03E9C81
,
0x34F4F86A
,
0xC69F7B69
,
0xD5CF889D
,
0x27A40B9E
,
0x79B737BA
,
0x8BDCB4B9
,
0x988C474D
,
0x6AE7C44E
,
0xBE2DA0A5
,
0x4C4623A6
,
0x5F16D052
,
0xAD7D5351
};
0xBE2DA0A5
,
0x4C4623A6
,
0x5F16D052
,
0xAD7D5351
};
static
const
uint32
crc_table_o40
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o40
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0x13A29877
,
0x274530EE
,
0x34E7A899
,
0x4E8A61DC
,
0x5D28F9AB
,
0x69CF5132
,
0x7A6DC945
,
0x9D14C3B8
,
0x00000000
,
0x13A29877
,
0x274530EE
,
0x34E7A899
,
0x4E8A61DC
,
0x5D28F9AB
,
0x69CF5132
,
0x7A6DC945
,
0x9D14C3B8
,
0x8EB65BCF
,
0xBA51F356
,
0xA9F36B21
,
0xD39EA264
,
0xC03C3A13
,
0xF4DB928A
,
0xE7790AFD
,
0x3FC5F181
,
0x2C6769F6
,
0x8EB65BCF
,
0xBA51F356
,
0xA9F36B21
,
0xD39EA264
,
0xC03C3A13
,
0xF4DB928A
,
0xE7790AFD
,
0x3FC5F181
,
0x2C6769F6
,
0x1880C16F
,
0x0B225918
,
0x714F905D
,
0x62ED082A
,
0x560AA0B3
,
0x45A838C4
,
0xA2D13239
,
0xB173AA4E
,
0x859402D7
,
0x1880C16F
,
0x0B225918
,
0x714F905D
,
0x62ED082A
,
0x560AA0B3
,
0x45A838C4
,
0xA2D13239
,
0xB173AA4E
,
0x859402D7
,
...
@@ -81,7 +80,7 @@ static const uint32 crc_table_o40[CRC_TABLE_SIZE] = {
...
@@ -81,7 +80,7 @@ static const uint32 crc_table_o40[CRC_TABLE_SIZE] = {
0x707D86E7
,
0x0A104FA2
,
0x19B2D7D5
,
0x2D557F4C
,
0x3EF7E73B
,
0xD98EEDC6
,
0xCA2C75B1
,
0xFECBDD28
,
0xED69455F
,
0x707D86E7
,
0x0A104FA2
,
0x19B2D7D5
,
0x2D557F4C
,
0x3EF7E73B
,
0xD98EEDC6
,
0xCA2C75B1
,
0xFECBDD28
,
0xED69455F
,
0x97048C1A
,
0x84A6146D
,
0xB041BCF4
,
0xA3E32483
};
0x97048C1A
,
0x84A6146D
,
0xB041BCF4
,
0xA3E32483
};
static
const
uint32
crc_table_o48
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o48
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0xA541927E
,
0x4F6F520D
,
0xEA2EC073
,
0x9EDEA41A
,
0x3B9F3664
,
0xD1B1F617
,
0x74F06469
,
0x38513EC5
,
0x00000000
,
0xA541927E
,
0x4F6F520D
,
0xEA2EC073
,
0x9EDEA41A
,
0x3B9F3664
,
0xD1B1F617
,
0x74F06469
,
0x38513EC5
,
0x9D10ACBB
,
0x773E6CC8
,
0xD27FFEB6
,
0xA68F9ADF
,
0x03CE08A1
,
0xE9E0C8D2
,
0x4CA15AAC
,
0x70A27D8A
,
0xD5E3EFF4
,
0x9D10ACBB
,
0x773E6CC8
,
0xD27FFEB6
,
0xA68F9ADF
,
0x03CE08A1
,
0xE9E0C8D2
,
0x4CA15AAC
,
0x70A27D8A
,
0xD5E3EFF4
,
0x3FCD2F87
,
0x9A8CBDF9
,
0xEE7CD990
,
0x4B3D4BEE
,
0xA1138B9D
,
0x045219E3
,
0x48F3434F
,
0xEDB2D131
,
0x079C1142
,
0x3FCD2F87
,
0x9A8CBDF9
,
0xEE7CD990
,
0x4B3D4BEE
,
0xA1138B9D
,
0x045219E3
,
0x48F3434F
,
0xEDB2D131
,
0x079C1142
,
...
@@ -112,7 +111,7 @@ static const uint32 crc_table_o48[CRC_TABLE_SIZE] = {
...
@@ -112,7 +111,7 @@ static const uint32 crc_table_o48[CRC_TABLE_SIZE] = {
0x378AB177
,
0x437AD51E
,
0xE63B4760
,
0x0C158713
,
0xA954156D
,
0xE5F54FC1
,
0x40B4DDBF
,
0xAA9A1DCC
,
0x0FDB8FB2
,
0x378AB177
,
0x437AD51E
,
0xE63B4760
,
0x0C158713
,
0xA954156D
,
0xE5F54FC1
,
0x40B4DDBF
,
0xAA9A1DCC
,
0x0FDB8FB2
,
0x7B2BEBDB
,
0xDE6A79A5
,
0x3444B9D6
,
0x91052BA8
};
0x7B2BEBDB
,
0xDE6A79A5
,
0x3444B9D6
,
0x91052BA8
};
static
const
uint32
crc_table_o56
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o56
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0xDD45AAB8
,
0xBF672381
,
0x62228939
,
0x7B2231F3
,
0xA6679B4B
,
0xC4451272
,
0x1900B8CA
,
0xF64463E6
,
0x00000000
,
0xDD45AAB8
,
0xBF672381
,
0x62228939
,
0x7B2231F3
,
0xA6679B4B
,
0xC4451272
,
0x1900B8CA
,
0xF64463E6
,
0x2B01C95E
,
0x49234067
,
0x9466EADF
,
0x8D665215
,
0x5023F8AD
,
0x32017194
,
0xEF44DB2C
,
0xE964B13D
,
0x34211B85
,
0x2B01C95E
,
0x49234067
,
0x9466EADF
,
0x8D665215
,
0x5023F8AD
,
0x32017194
,
0xEF44DB2C
,
0xE964B13D
,
0x34211B85
,
0x560392BC
,
0x8B463804
,
0x924680CE
,
0x4F032A76
,
0x2D21A34F
,
0xF06409F7
,
0x1F20D2DB
,
0xC2657863
,
0xA047F15A
,
0x560392BC
,
0x8B463804
,
0x924680CE
,
0x4F032A76
,
0x2D21A34F
,
0xF06409F7
,
0x1F20D2DB
,
0xC2657863
,
0xA047F15A
,
...
@@ -143,7 +142,7 @@ static const uint32 crc_table_o56[CRC_TABLE_SIZE] = {
...
@@ -143,7 +142,7 @@ static const uint32 crc_table_o56[CRC_TABLE_SIZE] = {
0xA565BA57
,
0xBC65029D
,
0x6120A825
,
0x0302211C
,
0xDE478BA4
,
0x31035088
,
0xEC46FA30
,
0x8E647309
,
0x5321D9B1
,
0xA565BA57
,
0xBC65029D
,
0x6120A825
,
0x0302211C
,
0xDE478BA4
,
0x31035088
,
0xEC46FA30
,
0x8E647309
,
0x5321D9B1
,
0x4A21617B
,
0x9764CBC3
,
0xF54642FA
,
0x2803E842
};
0x4A21617B
,
0x9764CBC3
,
0xF54642FA
,
0x2803E842
};
static
const
uint32
crc_table_o64
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o64
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0x38116FAC
,
0x7022DF58
,
0x4833B0F4
,
0xE045BEB0
,
0xD854D11C
,
0x906761E8
,
0xA8760E44
,
0xC5670B91
,
0x00000000
,
0x38116FAC
,
0x7022DF58
,
0x4833B0F4
,
0xE045BEB0
,
0xD854D11C
,
0x906761E8
,
0xA8760E44
,
0xC5670B91
,
0xFD76643D
,
0xB545D4C9
,
0x8D54BB65
,
0x2522B521
,
0x1D33DA8D
,
0x55006A79
,
0x6D1105D5
,
0x8F2261D3
,
0xB7330E7F
,
0xFD76643D
,
0xB545D4C9
,
0x8D54BB65
,
0x2522B521
,
0x1D33DA8D
,
0x55006A79
,
0x6D1105D5
,
0x8F2261D3
,
0xB7330E7F
,
0xFF00BE8B
,
0xC711D127
,
0x6F67DF63
,
0x5776B0CF
,
0x1F45003B
,
0x27546F97
,
0x4A456A42
,
0x725405EE
,
0x3A67B51A
,
0xFF00BE8B
,
0xC711D127
,
0x6F67DF63
,
0x5776B0CF
,
0x1F45003B
,
0x27546F97
,
0x4A456A42
,
0x725405EE
,
0x3A67B51A
,
...
@@ -174,7 +173,7 @@ static const uint32 crc_table_o64[CRC_TABLE_SIZE] = {
...
@@ -174,7 +173,7 @@ static const uint32 crc_table_o64[CRC_TABLE_SIZE] = {
0x854ADB82
,
0x2D3CD5C6
,
0x152DBA6A
,
0x5D1E0A9E
,
0x650F6532
,
0x081E60E7
,
0x300F0F4B
,
0x783CBFBF
,
0x402DD013
,
0x854ADB82
,
0x2D3CD5C6
,
0x152DBA6A
,
0x5D1E0A9E
,
0x650F6532
,
0x081E60E7
,
0x300F0F4B
,
0x783CBFBF
,
0x402DD013
,
0xE85BDE57
,
0xD04AB1FB
,
0x9879010F
,
0xA0686EA3
};
0xE85BDE57
,
0xD04AB1FB
,
0x9879010F
,
0xA0686EA3
};
static
const
uint32
crc_table_o72
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o72
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0xEF306B19
,
0xDB8CA0C3
,
0x34BCCBDA
,
0xB2F53777
,
0x5DC55C6E
,
0x697997B4
,
0x8649FCAD
,
0x6006181F
,
0x00000000
,
0xEF306B19
,
0xDB8CA0C3
,
0x34BCCBDA
,
0xB2F53777
,
0x5DC55C6E
,
0x697997B4
,
0x8649FCAD
,
0x6006181F
,
0x8F367306
,
0xBB8AB8DC
,
0x54BAD3C5
,
0xD2F32F68
,
0x3DC34471
,
0x097F8FAB
,
0xE64FE4B2
,
0xC00C303E
,
0x2F3C5B27
,
0x8F367306
,
0xBB8AB8DC
,
0x54BAD3C5
,
0xD2F32F68
,
0x3DC34471
,
0x097F8FAB
,
0xE64FE4B2
,
0xC00C303E
,
0x2F3C5B27
,
0x1B8090FD
,
0xF4B0FBE4
,
0x72F90749
,
0x9DC96C50
,
0xA975A78A
,
0x4645CC93
,
0xA00A2821
,
0x4F3A4338
,
0x7B8688E2
,
0x1B8090FD
,
0xF4B0FBE4
,
0x72F90749
,
0x9DC96C50
,
0xA975A78A
,
0x4645CC93
,
0xA00A2821
,
0x4F3A4338
,
0x7B8688E2
,
...
@@ -205,7 +204,7 @@ static const uint32 crc_table_o72[CRC_TABLE_SIZE] = {
...
@@ -205,7 +204,7 @@ static const uint32 crc_table_o72[CRC_TABLE_SIZE] = {
0x63480154
,
0xE501FDF9
,
0x0A3196E0
,
0x3E8D5D3A
,
0xD1BD3623
,
0x37F2D291
,
0xD8C2B988
,
0xEC7E7252
,
0x034E194B
,
0x63480154
,
0xE501FDF9
,
0x0A3196E0
,
0x3E8D5D3A
,
0xD1BD3623
,
0x37F2D291
,
0xD8C2B988
,
0xEC7E7252
,
0x034E194B
,
0x8507E5E6
,
0x6A378EFF
,
0x5E8B4525
,
0xB1BB2E3C
};
0x8507E5E6
,
0x6A378EFF
,
0x5E8B4525
,
0xB1BB2E3C
};
static
const
uint32
crc_table_o80
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o80
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0x68032CC8
,
0xD0065990
,
0xB8057558
,
0xA5E0C5D1
,
0xCDE3E919
,
0x75E69C41
,
0x1DE5B089
,
0x4E2DFD53
,
0x00000000
,
0x68032CC8
,
0xD0065990
,
0xB8057558
,
0xA5E0C5D1
,
0xCDE3E919
,
0x75E69C41
,
0x1DE5B089
,
0x4E2DFD53
,
0x262ED19B
,
0x9E2BA4C3
,
0xF628880B
,
0xEBCD3882
,
0x83CE144A
,
0x3BCB6112
,
0x53C84DDA
,
0x9C5BFAA6
,
0xF458D66E
,
0x262ED19B
,
0x9E2BA4C3
,
0xF628880B
,
0xEBCD3882
,
0x83CE144A
,
0x3BCB6112
,
0x53C84DDA
,
0x9C5BFAA6
,
0xF458D66E
,
0x4C5DA336
,
0x245E8FFE
,
0x39BB3F77
,
0x51B813BF
,
0xE9BD66E7
,
0x81BE4A2F
,
0xD27607F5
,
0xBA752B3D
,
0x02705E65
,
0x4C5DA336
,
0x245E8FFE
,
0x39BB3F77
,
0x51B813BF
,
0xE9BD66E7
,
0x81BE4A2F
,
0xD27607F5
,
0xBA752B3D
,
0x02705E65
,
...
@@ -236,7 +235,7 @@ static const uint32 crc_table_o80[CRC_TABLE_SIZE] = {
...
@@ -236,7 +235,7 @@ static const uint32 crc_table_o80[CRC_TABLE_SIZE] = {
0x96DC05CD
,
0x8B39B544
,
0xE33A998C
,
0x5B3FECD4
,
0x333CC01C
,
0x60F48DC6
,
0x08F7A10E
,
0xB0F2D456
,
0xD8F1F89E
,
0x96DC05CD
,
0x8B39B544
,
0xE33A998C
,
0x5B3FECD4
,
0x333CC01C
,
0x60F48DC6
,
0x08F7A10E
,
0xB0F2D456
,
0xD8F1F89E
,
0xC5144817
,
0xAD1764DF
,
0x15121187
,
0x7D113D4F
};
0xC5144817
,
0xAD1764DF
,
0x15121187
,
0x7D113D4F
};
static
const
uint32
crc_table_o88
[
CRC_TABLE_SIZE
]
=
{
static
const
uint32
_t
crc_table_o88
[
CRC_TABLE_SIZE
]
=
{
0x00000000
,
0x493C7D27
,
0x9278FA4E
,
0xDB448769
,
0x211D826D
,
0x6821FF4A
,
0xB3657823
,
0xFA590504
,
0x423B04DA
,
0x00000000
,
0x493C7D27
,
0x9278FA4E
,
0xDB448769
,
0x211D826D
,
0x6821FF4A
,
0xB3657823
,
0xFA590504
,
0x423B04DA
,
0x0B0779FD
,
0xD043FE94
,
0x997F83B3
,
0x632686B7
,
0x2A1AFB90
,
0xF15E7CF9
,
0xB86201DE
,
0x847609B4
,
0xCD4A7493
,
0x0B0779FD
,
0xD043FE94
,
0x997F83B3
,
0x632686B7
,
0x2A1AFB90
,
0xF15E7CF9
,
0xB86201DE
,
0x847609B4
,
0xCD4A7493
,
0x160EF3FA
,
0x5F328EDD
,
0xA56B8BD9
,
0xEC57F6FE
,
0x37137197
,
0x7E2F0CB0
,
0xC64D0D6E
,
0x8F717049
,
0x5435F720
,
0x160EF3FA
,
0x5F328EDD
,
0xA56B8BD9
,
0xEC57F6FE
,
0x37137197
,
0x7E2F0CB0
,
0xC64D0D6E
,
0x8F717049
,
0x5435F720
,
...
@@ -268,22 +267,22 @@ static const uint32 crc_table_o88[CRC_TABLE_SIZE] = {
...
@@ -268,22 +267,22 @@ static const uint32 crc_table_o88[CRC_TABLE_SIZE] = {
0xC451B7CC
,
0x8D6DCAEB
,
0x56294D82
,
0x1F1530A5
};
0xC451B7CC
,
0x8D6DCAEB
,
0x56294D82
,
0x1F1530A5
};
// Use the 8 table to calc crc32c value
// Use the 8 table to calc crc32c value
inline
void
CRC32T8
(
uint32
*
crc
,
const
uint8_t
**
p
)
{
inline
void
CRC32T8
(
uint32
_t
*
crc
,
const
uint8_t
**
p
)
{
auto
c
=
static_cast
<
uint32
>
(
*
crc
^
LE_LOAD32
(
*
p
));
auto
c
=
static_cast
<
uint32
_t
>
(
*
crc
^
LE_LOAD32
(
*
p
));
*
p
+=
4
;
*
p
+=
4
;
*
crc
=
crc_table_o88
[
c
&
0x
ff
]
^
crc_table_o80
[(
c
>>
8
)
&
0xff
]
^
crc_table_o72
[(
c
>>
16
)
&
0xff
]
^
*
crc
=
crc_table_o88
[
c
&
0x
FF
]
^
crc_table_o80
[(
c
>>
8
)
&
0xFF
]
^
crc_table_o72
[(
c
>>
16
)
&
0xFF
]
^
crc_table_o64
[(
c
>>
24
)
&
0x
ff
];
crc_table_o64
[(
c
>>
24
)
&
0x
FF
];
c
=
static_cast
<
uint32
>
(
LE_LOAD32
(
*
p
));
c
=
static_cast
<
uint32
_t
>
(
LE_LOAD32
(
*
p
));
*
crc
=
(
*
crc
)
^
crc_table_o56
[
c
&
0x
ff
]
^
crc_table_o48
[(
c
>>
8
)
&
0xff
]
^
crc_table_o40
[(
c
>>
16
)
&
0xff
]
^
*
crc
=
(
*
crc
)
^
crc_table_o56
[
c
&
0x
FF
]
^
crc_table_o48
[(
c
>>
8
)
&
0xFF
]
^
crc_table_o40
[(
c
>>
16
)
&
0xFF
]
^
crc_table_o32
[(
c
>>
24
)
&
0x
ff
];
crc_table_o32
[(
c
>>
24
)
&
0x
FF
];
*
p
+=
4
;
*
p
+=
4
;
}
}
// calc the crc32c value
// calc the crc32c value
uint32
MakeCrc32c
(
uint32
init_crc
,
const
char
*
data
,
size_t
size
)
{
uint32
_t
MakeCrc32c
(
uint32_t
init_crc
,
const
char
*
data
,
size_t
size
)
{
EXCEPT_CHECK_NULL
(
data
);
EXCEPT_CHECK_NULL
(
data
);
uint32
crc
=
init_crc
^
0xffffffffu
;
uint32
_t
crc
=
init_crc
^
0xFFFFFFFFU
;
const
unsigned
int
OFFSET
=
8
;
const
long
OFFSET
=
8
;
// Get the origin begin and end address(not alignment)
// Get the origin begin and end address(not alignment)
auto
*
bp
=
reinterpret_cast
<
const
uint8_t
*>
(
data
);
auto
*
bp
=
reinterpret_cast
<
const
uint8_t
*>
(
data
);
...
@@ -299,7 +298,7 @@ uint32 MakeCrc32c(uint32 init_crc, const char *data, size_t size) {
...
@@ -299,7 +298,7 @@ uint32 MakeCrc32c(uint32 init_crc, const char *data, size_t size) {
if
(
bp_align
<=
ep
)
{
if
(
bp_align
<=
ep
)
{
// Process bytes until finished or p is 4-byte aligned
// Process bytes until finished or p is 4-byte aligned
while
(
bp
!=
bp_align
)
{
while
(
bp
!=
bp_align
)
{
crc
=
crc_table_o32
[(
crc
&
0x
ff
)
^
(
*
bp
++
)]
^
(
crc
>>
8
);
crc
=
crc_table_o32
[(
crc
&
0x
FF
)
^
(
*
bp
++
)]
^
(
crc
>>
8
);
}
}
}
}
...
@@ -310,7 +309,7 @@ uint32 MakeCrc32c(uint32 init_crc, const char *data, size_t size) {
...
@@ -310,7 +309,7 @@ uint32 MakeCrc32c(uint32 init_crc, const char *data, size_t size) {
// Process the last not alignment bytes
// Process the last not alignment bytes
while
(
bp
<
ep
)
{
while
(
bp
<
ep
)
{
crc
=
crc_table_o32
[(
crc
&
0x
ff
)
^
(
*
bp
++
)]
^
(
crc
>>
8
);
crc
=
crc_table_o32
[(
crc
&
0x
FF
)
^
(
*
bp
++
)]
^
(
crc
>>
8
);
}
}
return
crc
^
0x
ffffffffu
;
return
crc
^
0x
FFFFFFFFU
;
}
}
mindinsight/datavisual/utils/crc32/crc32.h
浏览文件 @
5b455143
...
@@ -17,38 +17,66 @@
...
@@ -17,38 +17,66 @@
#ifndef DATAVISUAL_UTILS_CRC32_CRC32_H_
#ifndef DATAVISUAL_UTILS_CRC32_CRC32_H_
#define DATAVISUAL_UTILS_CRC32_CRC32_H_
#define DATAVISUAL_UTILS_CRC32_CRC32_H_
#include <pybind11/pybind11.h>
#include "pybind11/pybind11.h"
#include <stddef.h>
#include "securec/include/securec.h"
#include <cstddef>
#include <cstdint>
#include <cstdint>
#include "crc32/base.h"
#define CRC_TABLE_SIZE 256
#define RIGHT_SHIFT 15
#define LEFT_SHIFT 17
// Align n to (1 << m) byte boundary
// Align n to (1 << m) byte boundary
#define MEM_ALIGN(n, m) ((n + ((1 << m) - 1)) & ~((1 << m) - 1))
#define MEM_ALIGN(n, m) ((n + ((1 << m) - 1)) & ~((1 << m) - 1))
// check the null point, Only log it in if(): The value is null
#define EXCEPT_CHECK_NULL(value) \
do { \
if (value == nullptr) { \
break; \
} \
} while (0)
// implement common define function
// Get the 32 bits align value
inline
uint32_t
DecodeFixed32
(
const
char
*
ptr
)
{
uint32_t
result
=
0
;
if
(
EOK
!=
memcpy_s
(
&
result
,
sizeof
(
result
),
ptr
,
sizeof
(
result
)))
{
return
result
;
}
return
result
;
}
// Used to fetch a naturally-aligned 32-bit word in little endian byte-order
inline
uint32_t
LE_LOAD32
(
const
uint8_t
*
p
)
{
return
DecodeFixed32
(
reinterpret_cast
<
const
char
*>
(
p
));
}
// Masked for crc.
// Masked for crc.
static
constexpr
uint32
kMaskDelta
=
0xa282ead8ul
;
static
constexpr
uint32
_t
kMaskDelta
=
0xA282EAD8U
;
// Provide the Crc32c function
// Provide the Crc32c function
// Calculate the crc32c value, use the 8 table method
// Calculate the crc32c value, use the 8 table method
uint32
MakeCrc32c
(
uint32
init_crc
,
const
char
*
data
,
size_t
size
);
uint32
_t
MakeCrc32c
(
uint32_t
init_crc
,
const
char
*
data
,
size_t
size
);
uint32
GetMaskCrc32cValue
(
const
char
*
data
,
size_t
n
)
{
// A function return the crc32c value
auto
crc
=
MakeCrc32c
(
0
,
data
,
n
);
uint32_t
GetMaskCrc32cValue
(
const
char
*
data
,
size_t
n
)
{
return
crc
;
uint32_t
crc
=
MakeCrc32c
(
0
,
data
,
n
);
return
((
crc
>>
RIGHT_SHIFT
)
|
(
crc
<<
LEFT_SHIFT
))
+
kMaskDelta
;
}
}
uint32
GetValueFromStr
(
const
char
*
crc_str
)
{
// A function check the crc32c value against data
uint32
crc
=
DecodeFixed32
(
crc_str
);
bool
CheckValueAgainstData
(
const
char
*
crc_str
,
const
char
*
data
,
size_t
size
)
{
uint32
rot
=
crc
-
kMaskDelta
;
uint32_t
crc_new
=
GetMaskCrc32cValue
(
data
,
size
);
return
((
rot
>>
17
)
|
(
rot
<<
15
));
uint32_t
crc_old
=
DecodeFixed32
(
crc_str
);
return
crc_new
==
crc_old
;
}
}
PYBIND11_MODULE
(
crc32
,
m
)
{
PYBIND11_MODULE
(
crc32
,
m
)
{
m
.
doc
()
=
"crc util"
;
m
.
doc
()
=
"crc util"
;
m
.
def
(
"MakeCrc32c"
,
&
MakeCrc32c
,
"A function calculating the crc32c value, use the 8 table method"
);
m
.
def
(
"GetMaskCrc32cValue"
,
&
GetMaskCrc32cValue
,
"A function return the crc32c value"
);
m
.
def
(
"GetMaskCrc32cValue"
,
&
GetMaskCrc32cValue
,
"A function return the crc32c value"
);
m
.
def
(
"
GetValueFromStr"
,
&
GetValueFromStr
,
"A function return the crc32c value from string
"
);
m
.
def
(
"
CheckValueAgainstData"
,
&
CheckValueAgainstData
,
"A function check the crc32c value against data
"
);
}
}
#endif // DATAVISUAL_UTILS_CRC32_CRC32_H_
#endif // DATAVISUAL_UTILS_CRC32_CRC32_H_
mindinsight/lineagemgr/summary/event_writer.py
浏览文件 @
5b455143
...
@@ -19,8 +19,6 @@ import struct
...
@@ -19,8 +19,6 @@ import struct
from
mindinsight.datavisual.utils
import
crc32
from
mindinsight.datavisual.utils
import
crc32
KMASKDELTA
=
0xa282ead8
class
EventWriter
:
class
EventWriter
:
"""
"""
...
@@ -86,10 +84,6 @@ class EventWriter:
...
@@ -86,10 +84,6 @@ class EventWriter:
Returns:
Returns:
bytes, crc of content, 4 bytes.
bytes, crc of content, 4 bytes.
"""
"""
mask
=
(
1
<<
32
)
-
1
crc_value
=
crc32
.
GetMaskCrc32cValue
(
content
,
len
(
content
))
crc_value
=
crc32
.
MakeCrc32c
(
0
,
content
,
len
(
content
))
crc_value
=
((
crc_value
>>
15
)
|
(
crc_value
<<
17
))
&
mask
crc_value
=
(
crc_value
+
KMASKDELTA
)
&
mask
return
struct
.
pack
(
"<L"
,
crc_value
)
return
struct
.
pack
(
"<L"
,
crc_value
)
mindinsight/lineagemgr/summary/lineage_summary_analyzer.py
浏览文件 @
5b455143
...
@@ -137,8 +137,7 @@ class SummaryAnalyzer:
...
@@ -137,8 +137,7 @@ class SummaryAnalyzer:
Raises:
Raises:
LineageVerificationException: Raise when verification failed.
LineageVerificationException: Raise when verification failed.
"""
"""
if
crc32
.
GetValueFromStr
(
crc_str
)
!=
\
if
not
crc32
.
CheckValueAgainstData
(
crc_str
,
source_str
,
len
(
source_str
)):
crc32
.
GetMaskCrc32cValue
(
source_str
,
len
(
source_str
)):
log
.
error
(
"The CRC verification failed."
)
log
.
error
(
"The CRC verification failed."
)
raise
LineageVerificationException
(
"The CRC verification failed."
)
raise
LineageVerificationException
(
"The CRC verification failed."
)
...
...
tests/ut/datavisual/data_transform/test_data_manager.py
浏览文件 @
5b455143
...
@@ -51,8 +51,7 @@ class TestDataManager:
...
@@ -51,8 +51,7 @@ class TestDataManager:
@
pytest
.
fixture
(
scope
=
"function"
)
@
pytest
.
fixture
(
scope
=
"function"
)
def
crc_pass
(
self
):
def
crc_pass
(
self
):
"""Mock the crc to pass the check."""
"""Mock the crc to pass the check."""
ms_data_loader
.
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
ms_data_loader
.
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
ms_data_loader
.
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
def
_make_path_and_file_list
(
self
,
dir_name
):
def
_make_path_and_file_list
(
self
,
dir_name
):
"""Utils function for tests."""
"""Utils function for tests."""
...
...
tests/ut/datavisual/data_transform/test_ms_data_loader.py
浏览文件 @
5b455143
...
@@ -55,14 +55,12 @@ class TestMsDataLoader:
...
@@ -55,14 +55,12 @@ class TestMsDataLoader:
@
pytest
.
fixture
(
scope
=
"function"
)
@
pytest
.
fixture
(
scope
=
"function"
)
def
crc_pass
(
self
):
def
crc_pass
(
self
):
"""Mock the crc to pass the check."""
"""Mock the crc to pass the check."""
ms_data_loader
.
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
ms_data_loader
.
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
ms_data_loader
.
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
@
pytest
.
fixture
(
scope
=
"function"
)
@
pytest
.
fixture
(
scope
=
"function"
)
def
crc_fail
(
self
):
def
crc_fail
(
self
):
"""Mock the crc to fail the check."""
"""Mock the crc to fail the check."""
ms_data_loader
.
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
ms_data_loader
.
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
False
)
ms_data_loader
.
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
1
)
def
test_check_files_update_success_deleted_files
(
self
):
def
test_check_files_update_success_deleted_files
(
self
):
"""Test new file list delete some files."""
"""Test new file list delete some files."""
...
...
tests/ut/datavisual/processors/test_graph_processor.py
浏览文件 @
5b455143
...
@@ -55,8 +55,7 @@ class TestGraphProcessor:
...
@@ -55,8 +55,7 @@ class TestGraphProcessor:
@
classmethod
@
classmethod
def
setup_class
(
cls
):
def
setup_class
(
cls
):
"""Mock common environment for graph unittest."""
"""Mock common environment for graph unittest."""
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
data_manager
.
logger
=
MockLogger
data_manager
.
logger
=
MockLogger
def
teardown_class
(
self
):
def
teardown_class
(
self
):
...
...
tests/ut/datavisual/processors/test_images_processor.py
浏览文件 @
5b455143
...
@@ -57,8 +57,7 @@ class TestImagesProcessor:
...
@@ -57,8 +57,7 @@ class TestImagesProcessor:
@
classmethod
@
classmethod
def
setup_class
(
cls
):
def
setup_class
(
cls
):
"""Mock common environment for images unittest."""
"""Mock common environment for images unittest."""
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
data_manager
.
logger
=
MockLogger
data_manager
.
logger
=
MockLogger
def
teardown_class
(
self
):
def
teardown_class
(
self
):
...
...
tests/ut/datavisual/processors/test_scalars_processor.py
浏览文件 @
5b455143
...
@@ -54,8 +54,7 @@ class TestScalarsProcessor:
...
@@ -54,8 +54,7 @@ class TestScalarsProcessor:
@
classmethod
@
classmethod
def
setup_class
(
cls
):
def
setup_class
(
cls
):
"""Mock common environment for scalars unittest."""
"""Mock common environment for scalars unittest."""
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
data_manager
.
logger
=
MockLogger
data_manager
.
logger
=
MockLogger
def
teardown_class
(
self
):
def
teardown_class
(
self
):
...
...
tests/ut/datavisual/processors/test_train_task_manager.py
浏览文件 @
5b455143
...
@@ -59,8 +59,7 @@ class TestTrainTaskManager:
...
@@ -59,8 +59,7 @@ class TestTrainTaskManager:
"""
"""
Mock common environment for train task unittest.
Mock common environment for train task unittest.
"""
"""
crc32
.
GetValueFromStr
=
Mock
(
return_value
=
0
)
crc32
.
CheckValueAgainstData
=
Mock
(
return_value
=
True
)
crc32
.
GetMaskCrc32cValue
=
Mock
(
return_value
=
0
)
data_manager
.
logger
=
MockLogger
data_manager
.
logger
=
MockLogger
def
teardown_class
(
self
):
def
teardown_class
(
self
):
...
...
tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py
浏览文件 @
5b455143
...
@@ -93,8 +93,7 @@ class TestSummaryAnalyzer(TestCase):
...
@@ -93,8 +93,7 @@ class TestSummaryAnalyzer(TestCase):
def
test_check_crc
(
self
,
*
args
):
def
test_check_crc
(
self
,
*
args
):
"""Test _check_crc method."""
"""Test _check_crc method."""
args
[
0
].
return_value
=
None
args
[
0
].
return_value
=
None
args
[
1
].
GetValueFromStr
.
return_value
=
1
args
[
1
].
CheckValueAgainstData
.
return_value
=
False
args
[
1
].
GetMaskCrc32cValue
.
return_value
=
2
source_str
=
b
'
\x01
'
*
10
source_str
=
b
'
\x01
'
*
10
crc_str
=
b
'
\x00
'
*
4
crc_str
=
b
'
\x00
'
*
4
with
self
.
assertRaisesRegex
(
LineageVerificationException
,
"The CRC verification failed."
):
with
self
.
assertRaisesRegex
(
LineageVerificationException
,
"The CRC verification failed."
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录