Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Xiaomi
Mace
提交
1097312d
Mace
项目概览
Xiaomi
/
Mace
通知
106
Star
40
Fork
27
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
Mace
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
1097312d
编写于
5月 28, 2018
作者:
李
李寅
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Mock runtime failure
上级
2940ce5d
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
230 addition
and
50 deletion
+230
-50
mace/core/allocator.h
mace/core/allocator.h
+7
-0
mace/core/mace.cc
mace/core/mace.cc
+6
-3
mace/core/runtime/opencl/opencl_allocator.cc
mace/core/runtime/opencl/opencl_allocator.cc
+9
-0
mace/core/runtime_failure_mock.cc
mace/core/runtime_failure_mock.cc
+51
-0
mace/core/runtime_failure_mock.h
mace/core/runtime_failure_mock.h
+24
-0
mace/tools/validation/mace_run.cc
mace/tools/validation/mace_run.cc
+115
-45
tools/mace_tools.py
tools/mace_tools.py
+15
-2
tools/sh_commands.py
tools/sh_commands.py
+3
-0
未找到文件。
mace/core/allocator.h
浏览文件 @
1097312d
...
...
@@ -25,7 +25,9 @@
#include "mace/core/macros.h"
#include "mace/core/registry.h"
#include "mace/core/types.h"
#include "mace/core/runtime_failure_mock.h"
#include "mace/public/mace.h"
#include "mace/public/mace_runtime.h"
namespace
mace
{
...
...
@@ -65,6 +67,11 @@ class CPUAllocator : public Allocator {
if
(
nbytes
==
0
)
{
return
MaceStatus
::
MACE_SUCCESS
;
}
if
(
ShouldMockRuntimeFailure
())
{
return
MaceStatus
::
MACE_OUT_OF_RESOURCES
;
}
void
*
data
=
nullptr
;
#if defined(__ANDROID__) || defined(__hexagon__)
data
=
memalign
(
kMaceAlignment
,
nbytes
);
...
...
mace/core/mace.cc
浏览文件 @
1097312d
...
...
@@ -121,14 +121,16 @@ MaceEngine::Impl::Impl(DeviceType device_type)
#ifdef MACE_ENABLE_HEXAGON
,
hexagon_controller_
(
nullptr
)
#endif
{}
{
LOG
(
INFO
)
<<
"Creating MaceEngine, MACE version: "
<<
MaceVersion
();
}
MaceStatus
MaceEngine
::
Impl
::
Init
(
const
NetDef
*
net_def
,
const
std
::
vector
<
std
::
string
>
&
input_nodes
,
const
std
::
vector
<
std
::
string
>
&
output_nodes
,
const
unsigned
char
*
model_data
)
{
LOG
(
INFO
)
<<
"
MACE version: "
<<
MaceVersion
()
;
LOG
(
INFO
)
<<
"
Initializing MaceEngine"
;
// Set storage path for internal usage
for
(
auto
input_name
:
input_nodes
)
{
ws_
->
CreateTensor
(
MakeString
(
"mace_input_node_"
,
input_name
),
...
...
@@ -158,7 +160,7 @@ MaceStatus MaceEngine::Impl::Init(
MACE_RETURN_IF_ERROR
(
ws_
->
LoadModelTensor
(
*
net_def
,
device_type_
,
model_data
));
// Init model
// Init model
auto
net
=
CreateNet
(
op_registry_
,
*
net_def
,
ws_
.
get
(),
device_type_
,
NetMode
::
INIT
);
MACE_RETURN_IF_ERROR
(
net
->
Run
());
...
...
@@ -170,6 +172,7 @@ MaceStatus MaceEngine::Impl::Init(
}
MaceEngine
::
Impl
::~
Impl
()
{
LOG
(
INFO
)
<<
"Destroying MaceEngine"
;
#ifdef MACE_ENABLE_HEXAGON
if
(
device_type_
==
HEXAGON
)
{
if
(
VLOG_IS_ON
(
2
))
{
...
...
mace/core/runtime/opencl/opencl_allocator.cc
浏览文件 @
1097312d
...
...
@@ -45,6 +45,11 @@ MaceStatus OpenCLAllocator::New(size_t nbytes, void **result) const {
return
MaceStatus
::
MACE_SUCCESS
;
}
VLOG
(
3
)
<<
"Allocate OpenCL buffer: "
<<
nbytes
;
if
(
ShouldMockRuntimeFailure
())
{
return
MaceStatus
::
MACE_OUT_OF_RESOURCES
;
}
cl_int
error
;
cl
::
Buffer
*
buffer
=
new
cl
::
Buffer
(
OpenCLRuntime
::
Global
()
->
context
(),
CL_MEM_READ_WRITE
|
CL_MEM_ALLOC_HOST_PTR
,
...
...
@@ -68,6 +73,10 @@ MaceStatus OpenCLAllocator::NewImage(const std::vector<size_t> &image_shape,
VLOG
(
3
)
<<
"Allocate OpenCL image: "
<<
image_shape
[
0
]
<<
", "
<<
image_shape
[
1
];
if
(
ShouldMockRuntimeFailure
())
{
return
MaceStatus
::
MACE_OUT_OF_RESOURCES
;
}
cl
::
ImageFormat
img_format
(
CL_RGBA
,
DataTypeToCLChannelType
(
dt
));
cl_int
error
;
...
...
mace/core/runtime_failure_mock.cc
0 → 100644
浏览文件 @
1097312d
// Copyright 2018 Xiaomi, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdlib>
#include <string>
#include "mace/core/runtime_failure_mock.h"
#include "mace/utils/logging.h"
namespace
mace
{
namespace
{
inline
float
GetRuntimeFailureRatioFromEnv
()
{
const
char
*
env
=
getenv
(
"MACE_RUNTIME_FAILURE_RATIO"
);
if
(
env
==
nullptr
)
{
return
0
;
}
std
::
string
env_str
(
env
);
std
::
istringstream
ss
(
env_str
);
float
ratio
;
ss
>>
ratio
;
return
ratio
;
}
}
// namespace
bool
ShouldMockRuntimeFailure
()
{
static
unsigned
int
seed
=
time
(
NULL
);
static
float
mock_runtime_failure_ratio
=
GetRuntimeFailureRatioFromEnv
();
if
(
mock_runtime_failure_ratio
>
1e-6
)
{
float
random_ratio
=
rand_r
(
&
seed
)
/
static_cast
<
float
>
(
RAND_MAX
);
if
(
random_ratio
<
mock_runtime_failure_ratio
)
{
VLOG
(
0
)
<<
"Mock runtime failure."
;
return
true
;
}
}
return
false
;
}
}
// namespace mace
mace/core/runtime_failure_mock.h
0 → 100644
浏览文件 @
1097312d
// Copyright 2018 Xiaomi, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MACE_CORE_RUNTIME_FAILURE_MOCK_H_
#define MACE_CORE_RUNTIME_FAILURE_MOCK_H_
namespace
mace
{
bool
ShouldMockRuntimeFailure
();
}
// namespace mace
#endif // MACE_CORE_RUNTIME_FAILURE_MOCK_H_
mace/tools/validation/mace_run.cc
浏览文件 @
1097312d
...
...
@@ -120,45 +120,48 @@ struct mallinfo LogMallinfoChange(struct mallinfo prev) {
struct
mallinfo
curr
=
mallinfo
();
if
(
prev
.
arena
!=
curr
.
arena
)
{
LOG
(
INFO
)
<<
"Non-mmapped space allocated (bytes): "
<<
curr
.
arena
<<
", diff: "
<<
((
int64_t
)
curr
.
arena
-
(
int64_t
)
prev
.
arena
);
<<
", diff: "
<<
((
int64_t
)
curr
.
arena
-
(
int64_t
)
prev
.
arena
);
}
if
(
prev
.
ordblks
!=
curr
.
ordblks
)
{
LOG
(
INFO
)
<<
"Number of free chunks: "
<<
curr
.
ordblks
<<
", diff: "
<<
((
int64_t
)
curr
.
ordblks
-
(
int64_t
)
prev
.
ordblks
);
<<
", diff: "
<<
((
int64_t
)
curr
.
ordblks
-
(
int64_t
)
prev
.
ordblks
);
}
if
(
prev
.
smblks
!=
curr
.
smblks
)
{
LOG
(
INFO
)
<<
"Number of free fastbin blocks: "
<<
curr
.
smblks
<<
", diff: "
<<
((
int64_t
)
curr
.
smblks
-
(
int64_t
)
prev
.
smblks
);
<<
", diff: "
<<
((
int64_t
)
curr
.
smblks
-
(
int64_t
)
prev
.
smblks
);
}
if
(
prev
.
hblks
!=
curr
.
hblks
)
{
LOG
(
INFO
)
<<
"Number of mmapped regions: "
<<
curr
.
hblks
<<
", diff: "
<<
((
int64_t
)
curr
.
hblks
-
(
int64_t
)
prev
.
hblks
);
<<
", diff: "
<<
((
int64_t
)
curr
.
hblks
-
(
int64_t
)
prev
.
hblks
);
}
if
(
prev
.
hblkhd
!=
curr
.
hblkhd
)
{
LOG
(
INFO
)
<<
"Space allocated in mmapped regions (bytes): "
<<
curr
.
hblkhd
<<
", diff: "
<<
((
int64_t
)
curr
.
hblkhd
-
(
int64_t
)
prev
.
hblkhd
);
<<
", diff: "
<<
((
int64_t
)
curr
.
hblkhd
-
(
int64_t
)
prev
.
hblkhd
);
}
if
(
prev
.
usmblks
!=
curr
.
usmblks
)
{
LOG
(
INFO
)
<<
"Maximum total allocated space (bytes): "
<<
curr
.
usmblks
<<
", diff: "
<<
((
int64_t
)
curr
.
usmblks
-
(
int64_t
)
prev
.
usmblks
);
<<
", diff: "
<<
((
int64_t
)
curr
.
usmblks
-
(
int64_t
)
prev
.
usmblks
);
}
if
(
prev
.
fsmblks
!=
curr
.
fsmblks
)
{
LOG
(
INFO
)
<<
"Space in freed fastbin blocks (bytes): "
<<
curr
.
fsmblks
<<
", diff: "
<<
((
int64_t
)
curr
.
fsmblks
-
(
int64_t
)
prev
.
fsmblks
);
<<
", diff: "
<<
((
int64_t
)
curr
.
fsmblks
-
(
int64_t
)
prev
.
fsmblks
);
}
if
(
prev
.
uordblks
!=
curr
.
uordblks
)
{
LOG
(
INFO
)
<<
"Total allocated space (bytes): "
<<
curr
.
uordblks
<<
", diff: "
<<
((
int64_t
)
curr
.
uordblks
-
(
int64_t
)
prev
.
uordblks
);
<<
((
int64_t
)
curr
.
uordblks
-
(
int64_t
)
prev
.
uordblks
);
}
if
(
prev
.
fordblks
!=
curr
.
fordblks
)
{
LOG
(
INFO
)
<<
"Total free space (bytes): "
<<
curr
.
fordblks
<<
", diff: "
<<
((
int64_t
)
curr
.
fordblks
-
(
int64_t
)
prev
.
fordblks
);
<<
((
int64_t
)
curr
.
fordblks
-
(
int64_t
)
prev
.
fordblks
);
}
if
(
prev
.
keepcost
!=
curr
.
keepcost
)
{
LOG
(
INFO
)
<<
"Top-most, releasable space (bytes): "
<<
curr
.
keepcost
<<
", diff: "
<<
((
int64_t
)
curr
.
keepcost
-
(
int64_t
)
prev
.
keepcost
);
<<
((
int64_t
)
curr
.
keepcost
-
(
int64_t
)
prev
.
keepcost
);
}
return
curr
;
}
...
...
@@ -227,39 +230,48 @@ bool RunModel(const std::string &model_name,
new
FileStorageFactory
(
kernel_file_path
));
SetKVStorageFactory
(
storage_factory
);
std
::
shared_ptr
<
mace
::
MaceEngine
>
engine
;
MaceStatus
create_engine_status
;
// Create Engine
int64_t
t0
=
NowMicros
();
std
::
vector
<
unsigned
char
>
model_pb_data
;
if
(
FLAGS_model_file
!=
""
)
{
std
::
vector
<
unsigned
char
>
model_pb_data
;
if
(
!
mace
::
ReadBinaryFile
(
&
model_pb_data
,
FLAGS_model_file
))
{
LOG
(
FATAL
)
<<
"Failed to read file: "
<<
FLAGS_model_file
;
}
create_engine_status
=
CreateMaceEngineFromProto
(
model_pb_data
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
else
{
create_engine_status
=
CreateMaceEngineFromCode
(
model_name
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
int64_t
t1
=
NowMicros
();
if
(
create_engine_status
!=
MaceStatus
::
MACE_SUCCESS
)
{
LOG
(
FATAL
)
<<
"Create engine error, please check the arguments"
;
}
std
::
shared_ptr
<
mace
::
MaceEngine
>
engine
;
MaceStatus
create_engine_status
;
double
init_millis
=
(
t1
-
t0
)
/
1000.0
;
LOG
(
INFO
)
<<
"Total init latency: "
<<
init_millis
<<
" ms"
;
double
init_millis
;
while
(
true
)
{
// Create Engine
int64_t
t0
=
NowMicros
();
if
(
FLAGS_model_file
!=
""
)
{
create_engine_status
=
CreateMaceEngineFromProto
(
model_pb_data
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
else
{
create_engine_status
=
CreateMaceEngineFromCode
(
model_name
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
int64_t
t1
=
NowMicros
();
if
(
create_engine_status
!=
MACE_SUCCESS
)
{
LOG
(
ERROR
)
<<
"Create engine runtime error, retry ... errcode: "
<<
create_engine_status
;
}
else
{
init_millis
=
(
t1
-
t0
)
/
1000.0
;
LOG
(
INFO
)
<<
"Total init latency: "
<<
init_millis
<<
" ms"
;
break
;
}
}
const
size_t
input_count
=
input_names
.
size
();
const
size_t
output_count
=
output_names
.
size
();
...
...
@@ -297,26 +309,84 @@ bool RunModel(const std::string &model_name,
}
LOG
(
INFO
)
<<
"Warm up run"
;
int64_t
t3
=
NowMicros
();
engine
->
Run
(
inputs
,
&
outputs
);
int64_t
t4
=
NowMicros
();
double
warmup_millis
=
(
t4
-
t3
)
/
1000.0
;
LOG
(
INFO
)
<<
"1st warm up run latency: "
<<
warmup_millis
<<
" ms"
;
double
warmup_millis
;
while
(
true
)
{
int64_t
t3
=
NowMicros
();
MaceStatus
warmup_status
=
engine
->
Run
(
inputs
,
&
outputs
);
if
(
warmup_status
!=
MACE_SUCCESS
)
{
LOG
(
ERROR
)
<<
"Warmup runtime error, retry ... errcode: "
<<
warmup_status
;
do
{
if
(
FLAGS_model_file
!=
""
)
{
create_engine_status
=
CreateMaceEngineFromProto
(
model_pb_data
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
else
{
create_engine_status
=
CreateMaceEngineFromCode
(
model_name
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
}
while
(
create_engine_status
!=
MACE_SUCCESS
);
}
else
{
int64_t
t4
=
NowMicros
();
warmup_millis
=
(
t4
-
t3
)
/
1000.0
;
LOG
(
INFO
)
<<
"1st warm up run latency: "
<<
warmup_millis
<<
" ms"
;
break
;
}
}
double
model_run_millis
=
-
1
;
if
(
FLAGS_round
>
0
)
{
LOG
(
INFO
)
<<
"Run model"
;
int64_t
t
0
=
NowMicros
()
;
int64_t
t
otal_run_duration
=
0
;
struct
mallinfo
prev
=
mallinfo
();
for
(
int
i
=
0
;
i
<
FLAGS_round
;
++
i
)
{
engine
->
Run
(
inputs
,
&
outputs
);
MaceStatus
run_status
;
while
(
true
)
{
int64_t
t0
=
NowMicros
();
run_status
=
engine
->
Run
(
inputs
,
&
outputs
);
if
(
run_status
!=
MACE_SUCCESS
)
{
LOG
(
ERROR
)
<<
"Mace run model runtime error, retry ... errcode: "
<<
run_status
;
do
{
if
(
FLAGS_model_file
!=
""
)
{
create_engine_status
=
CreateMaceEngineFromProto
(
model_pb_data
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
else
{
create_engine_status
=
CreateMaceEngineFromCode
(
model_name
,
FLAGS_model_data_file
,
input_names
,
output_names
,
device_type
,
&
engine
);
}
}
while
(
create_engine_status
!=
MACE_SUCCESS
);
}
else
{
int64_t
t1
=
NowMicros
();
total_run_duration
+=
(
t1
-
t0
);
break
;
}
}
if
(
FLAGS_malloc_check_cycle
>=
1
&&
i
%
FLAGS_malloc_check_cycle
==
0
)
{
LOG
(
INFO
)
<<
"=== check malloc info change #"
<<
i
<<
" ==="
;
prev
=
LogMallinfoChange
(
prev
);
}
}
int64_t
t1
=
NowMicros
();
model_run_millis
=
(
t1
-
t0
)
/
1000.0
/
FLAGS_round
;
model_run_millis
=
total_run_duration
/
1000.0
/
FLAGS_round
;
LOG
(
INFO
)
<<
"Average latency: "
<<
model_run_millis
<<
" ms"
;
}
...
...
tools/mace_tools.py
浏览文件 @
1097312d
...
...
@@ -171,7 +171,8 @@ def tuning_run(target_abi,
omp_num_threads
=-
1
,
cpu_affinity_policy
=
1
,
gpu_perf_hint
=
3
,
gpu_priority_hint
=
3
):
gpu_priority_hint
=
3
,
runtime_failure_ratio
=
0.0
):
stdout
=
sh_commands
.
tuning_run
(
target_abi
,
serialno
,
...
...
@@ -195,6 +196,7 @@ def tuning_run(target_abi,
cpu_affinity_policy
,
gpu_perf_hint
,
gpu_priority_hint
,
runtime_failure_ratio
,
valgrind
=
FLAGS
.
valgrind
,
valgrind_path
=
FLAGS
.
valgrind_path
,
valgrind_args
=
FLAGS
.
valgrind_args
...
...
@@ -543,6 +545,11 @@ def parse_args():
type
=
str
,
default
=
"half"
,
help
=
"[half | float]."
)
parser
.
add_argument
(
"--runtime_failure_ratio"
,
type
=
float
,
default
=
0.0
,
help
=
"[mock runtime failure ratio]."
)
return
parser
.
parse_known_args
()
...
...
@@ -632,6 +639,11 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
if
FLAGS
.
mode
==
"run"
or
FLAGS
.
mode
==
"validate"
or
\
FLAGS
.
mode
==
"all"
:
if
FLAGS
.
mode
==
"run"
:
runtime_failure_ratio
=
FLAGS
.
runtime_failure_ratio
else
:
runtime_failure_ratio
=
0.0
tuning_run
(
target_abi
,
serialno
,
vlog_level
,
...
...
@@ -651,7 +663,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
omp_num_threads
=
FLAGS
.
omp_num_threads
,
cpu_affinity_policy
=
FLAGS
.
cpu_affinity_policy
,
gpu_perf_hint
=
FLAGS
.
gpu_perf_hint
,
gpu_priority_hint
=
FLAGS
.
gpu_priority_hint
)
gpu_priority_hint
=
FLAGS
.
gpu_priority_hint
,
runtime_failure_ratio
=
runtime_failure_ratio
)
if
FLAGS
.
mode
==
"benchmark"
:
gen_opencl_and_tuning_code
(
...
...
tools/sh_commands.py
浏览文件 @
1097312d
...
...
@@ -597,6 +597,7 @@ def tuning_run(abi,
cpu_affinity_policy
=
1
,
gpu_perf_hint
=
3
,
gpu_priority_hint
=
3
,
runtime_failure_ratio
=
0.0
,
valgrind
=
False
,
valgrind_path
=
"/data/local/tmp/valgrind"
,
valgrind_args
=
""
,
...
...
@@ -617,6 +618,7 @@ def tuning_run(abi,
[
"env"
,
"MACE_CPP_MIN_VLOG_LEVEL=%s"
%
vlog_level
,
"MACE_RUNTIME_FAILURE_RATIO=%f"
%
runtime_failure_ratio
,
"%s/mace_run"
%
model_output_dir
,
"--model_name=%s"
%
model_tag
,
"--input_node=%s"
%
","
.
join
(
input_nodes
),
...
...
@@ -678,6 +680,7 @@ def tuning_run(abi,
"MACE_RUN_PARAMETER_PATH=%s/mace_run.config"
%
phone_data_dir
,
"MACE_INTERNAL_STORAGE_PATH=%s"
%
internal_storage_dir
,
"MACE_LIMIT_OPENCL_KERNEL_TIME=%s"
%
limit_opencl_kernel_time
,
"MACE_RUNTIME_FAILURE_RATIO=%f"
%
runtime_failure_ratio
,
]
if
valgrind
:
adb_cmd
.
extend
([
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录