Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
闹闹不闹!
Mace
提交
39fb3055
Mace
项目概览
闹闹不闹!
/
Mace
与 Fork 源项目一致
Fork自
Xiaomi / Mace
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
Mace
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
39fb3055
编写于
6月 04, 2018
作者:
L
liuqi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refactor opencl binary load logic: use file instead of code.
上级
d4124708
变更
15
隐藏空白更改
内联
并排
Showing
15 changed file
with
284 addition
and
279 deletion
+284
-279
docs/getting_started/how_to_build.rst
docs/getting_started/how_to_build.rst
+22
-5
docs/getting_started/how_to_build_zh.rst
docs/getting_started/how_to_build_zh.rst
+7
-0
mace/benchmark/benchmark_model.cc
mace/benchmark/benchmark_model.cc
+8
-0
mace/core/runtime/opencl/opencl_runtime.cc
mace/core/runtime/opencl/opencl_runtime.cc
+110
-36
mace/core/runtime/opencl/opencl_runtime.h
mace/core/runtime/opencl/opencl_runtime.h
+8
-3
mace/examples/example.cc
mace/examples/example.cc
+7
-0
mace/public/mace_runtime.h
mace/public/mace_runtime.h
+8
-1
mace/python/tools/opencl_codegen.py
mace/python/tools/opencl_codegen.py
+0
-124
mace/python/tools/opencl_compiled_kernel.cc.jinja2
mace/python/tools/opencl_compiled_kernel.cc.jinja2
+0
-39
mace/tools/validation/BUILD
mace/tools/validation/BUILD
+0
-1
mace/tools/validation/mace_run.cc
mace/tools/validation/mace_run.cc
+8
-24
tools/bazel_adb_run.py
tools/bazel_adb_run.py
+0
-1
tools/common.py
tools/common.py
+9
-0
tools/converter.py
tools/converter.py
+27
-22
tools/sh_commands.py
tools/sh_commands.py
+70
-23
未找到文件。
docs/getting_started/how_to_build.rst
浏览文件 @
39fb3055
...
...
@@ -365,6 +365,16 @@ The followings list the details.
``.pb`` file will be generated only when build_type is ``proto``.
**OpenCL compiled kernel binary file**
* ``opencl/compiled_kernel.bin``
.. note::
This file will be generated only when specify ``target_soc`` and runtime is ``gpu``.
.. warning::
This file rely on the OpenCL driver on the phone, you should update the file when OpenCL driver changed.
=============
5. how to use
...
...
@@ -385,14 +395,21 @@ Please refer to \ ``mace/examples/example.cc``\ for full usage. the following li
new FileStorageFactory(file_path));
ConfigKVStorageFactory(storage_factory);
//1. Declare the device type(must be same with ``runtime`` in configuration file)
// 1. set precompiled OpenCL binary file paths if you use gpu of specified SOC,
// Besides the binary rely on the OpenCL driver of the SOC,
// if OpenCL driver changed, you should recompiled the binary file.
if (device_type == DeviceType::GPU) {
mace::SetOpenCLBinaryPaths(opencl_binary_paths);
}
// 2. Declare the device type(must be same with ``runtime`` in configuration file)
DeviceType device_type = DeviceType::GPU;
//
2
. Define the input and output tensor names.
//
3
. Define the input and output tensor names.
std::vector<std::string> input_names = {...};
std::vector<std::string> output_names = {...};
//
3
. Create MaceEngine object
//
4
. Create MaceEngine object
std::shared_ptr<mace::MaceEngine> engine;
MaceStatus create_engine_status;
// Create Engine from code
...
...
@@ -415,7 +432,7 @@ Please refer to \ ``mace/examples/example.cc``\ for full usage. the following li
// do something
}
//
4
. Create Input and Output objects
//
5
. Create Input and Output objects
std::map<std::string, mace::MaceTensor> inputs;
std::map<std::string, mace::MaceTensor> outputs;
for (size_t i = 0; i < input_count; ++i) {
...
...
@@ -440,6 +457,6 @@ Please refer to \ ``mace/examples/example.cc``\ for full usage. the following li
outputs[output_names[i]] = mace::MaceTensor(output_shapes[i], buffer_out);
}
//
5
. Run the model
//
6
. Run the model
MaceStatus status = engine.Run(inputs, &outputs);
docs/getting_started/how_to_build_zh.rst
浏览文件 @
39fb3055
...
...
@@ -390,6 +390,13 @@ Mace目前只提供静态库,有以下两种使用场景。
new FileStorageFactory(file_path));
ConfigKVStorageFactory(storage_factory);
// 2. 如果你使用特定SOC的GPU,可以设置OpenCL预编译的二进制文件路径。
// * 该二进制文件是依赖于手机上OpenCL driver的,如果OpenCL driver改变了,
// 你需要重新编译并更新该二进制文件。
if (device_type == DeviceType::GPU) {
mace::SetOpenCLBinaryPaths(opencl_binary_paths);
}
//1. 声明设备类型(必须与build时指定的runtime一致)
DeviceType device_type = DeviceType::GPU;
...
...
mace/benchmark/benchmark_model.cc
浏览文件 @
39fb3055
...
...
@@ -188,6 +188,9 @@ DEFINE_string(input_file, "", "input file name");
DEFINE_int32
(
max_num_runs
,
100
,
"number of runs max"
);
DEFINE_string
(
max_time
,
"10.0"
,
"length to run max"
);
DEFINE_int32
(
warmup_runs
,
1
,
"how many runs to initialize model"
);
DEFINE_string
(
opencl_binary_file
,
""
,
"compiled opencl binary file path"
);
DEFINE_string
(
model_data_file
,
""
,
"model data file name, used when EMBED_MODEL_DATA set to 0"
);
DEFINE_string
(
model_file
,
""
,
...
...
@@ -270,6 +273,11 @@ int Main(int argc, char **argv) {
new
FileStorageFactory
(
kernel_file_path
));
SetKVStorageFactory
(
storage_factory
);
if
(
device_type
==
DeviceType
::
GPU
)
{
std
::
vector
<
std
::
string
>
opencl_binary_paths
=
{
FLAGS_opencl_binary_file
};
mace
::
SetOpenCLBinaryPaths
(
opencl_binary_paths
);
}
// Create Engine
std
::
shared_ptr
<
mace
::
MaceEngine
>
engine
;
MaceStatus
create_engine_status
;
...
...
mace/core/runtime/opencl/opencl_runtime.cc
浏览文件 @
39fb3055
...
...
@@ -14,6 +14,8 @@
#include "mace/core/runtime/opencl/opencl_runtime.h"
#include <sys/stat.h>
#include <cstdlib>
#include <fstream>
#include <memory>
...
...
@@ -31,9 +33,6 @@
namespace
mace
{
extern
const
std
::
map
<
std
::
string
,
std
::
vector
<
unsigned
char
>>
kCompiledProgramMap
;
extern
const
std
::
string
kCompiledProgramPlatform
;
extern
const
std
::
map
<
std
::
string
,
std
::
vector
<
unsigned
char
>>
kEncryptedProgramMap
;
...
...
@@ -43,6 +42,12 @@ void SetGPUHints(GPUPerfHint gpu_perf_hint, GPUPriorityHint gpu_priority_hint) {
OpenCLRuntime
::
Configure
(
gpu_perf_hint
,
gpu_priority_hint
);
}
// Set OpenCL Compiled Binary paths, just call once. (Not thread-safe)
void
SetOpenCLBinaryPaths
(
const
std
::
vector
<
std
::
string
>
&
paths
)
{
OpenCLRuntime
::
ConfigureOpenCLBinaryPath
(
paths
);
}
const
std
::
string
OpenCLErrorToString
(
cl_int
error
)
{
switch
(
error
)
{
case
CL_SUCCESS
:
...
...
@@ -237,6 +242,25 @@ GPUType ParseGPUType(const std::string &device_name) {
return
GPUType
::
UNKNOWN
;
}
}
std
::
string
FindFirstExistPath
(
const
std
::
vector
<
std
::
string
>
&
paths
)
{
std
::
string
result
;
struct
stat
st
;
for
(
auto
path
:
paths
)
{
if
(
stat
(
path
.
c_str
(),
&
st
)
==
0
)
{
if
(
S_ISREG
(
st
.
st_mode
))
{
result
=
path
;
break
;
}
}
}
return
result
;
}
const
char
*
kOpenCLPlatformInfoKey
=
"mace_opencl_precompiled_platform_info_key"
;
const
char
*
kPrecompiledProgramFileName
=
"mace_cl_compiled_program.bin"
;
}
// namespace
void
OpenCLProfilingTimer
::
StartTiming
()
{}
...
...
@@ -267,6 +291,8 @@ void OpenCLProfilingTimer::ClearTiming() {
GPUPerfHint
OpenCLRuntime
::
kGPUPerfHint
=
GPUPerfHint
::
PERF_NORMAL
;
GPUPriorityHint
OpenCLRuntime
::
kGPUPriorityHint
=
GPUPriorityHint
::
PRIORITY_DEFAULT
;
std
::
string
OpenCLRuntime
::
kPrecompiledBinaryPath
=
""
;
// NOLINT(runtime/string)
OpenCLRuntime
*
OpenCLRuntime
::
Global
()
{
static
OpenCLRuntime
runtime
;
...
...
@@ -279,9 +305,19 @@ void OpenCLRuntime::Configure(GPUPerfHint gpu_perf_hint,
OpenCLRuntime
::
kGPUPriorityHint
=
gpu_priority_hint
;
}
void
OpenCLRuntime
::
ConfigureOpenCLBinaryPath
(
const
std
::
vector
<
std
::
string
>
&
paths
)
{
OpenCLRuntime
::
kPrecompiledBinaryPath
=
FindFirstExistPath
(
paths
);
if
(
OpenCLRuntime
::
kPrecompiledBinaryPath
.
empty
())
{
LOG
(
WARNING
)
<<
"There is no precompiled OpenCL binary file in "
<<
MakeString
(
paths
);
}
}
OpenCLRuntime
::
OpenCLRuntime
()
:
storage_
(
nullptr
),
is_profiling_enabled_
(
false
)
{
precompiled_binary_storage_
(
nullptr
),
cache_storage_
(
nullptr
),
is_profiling_enabled_
(
false
)
{
LoadOpenCLLibrary
();
std
::
vector
<
cl
::
Platform
>
all_platforms
;
...
...
@@ -369,12 +405,38 @@ OpenCLRuntime::OpenCLRuntime():
extern
std
::
shared_ptr
<
KVStorageFactory
>
kStorageFactory
;
if
(
kStorageFactory
!=
nullptr
)
{
c
onst
std
::
string
cl_compiled_file_name
=
"mace_cl_compiled_program.bin"
;
storage_
=
kStorageFactory
->
CreateStorage
(
cl_compiled_file_n
ame
);
c
ache_storage_
=
kStorageFactory
->
CreateStorage
(
kPrecompiledProgramFileN
ame
);
if
(
platform_info_
!=
kCompiledProgramPlatform
)
{
if
(
storage_
->
Load
()
!=
0
)
{
LOG
(
FATAL
)
<<
"Load opencl compiled kernel file failed"
;
if
(
cache_storage_
->
Load
()
!=
0
)
{
LOG
(
FATAL
)
<<
"Load OpenCL cached compiled kernel file failed"
;
}
auto
platform_info_array
=
this
->
cache_storage_
->
Find
(
kOpenCLPlatformInfoKey
);
if
(
platform_info_array
!=
nullptr
)
{
cached_binary_platform_info_
=
std
::
string
(
platform_info_array
->
begin
(),
platform_info_array
->
end
());
}
}
if
(
cached_binary_platform_info_
!=
platform_info_
)
{
if
(
OpenCLRuntime
::
kPrecompiledBinaryPath
.
empty
())
{
LOG
(
WARNING
)
<<
"There is no precompiled OpenCL binary in"
" all OpenCL binary paths"
;
}
else
{
precompiled_binary_storage_
.
reset
(
new
FileStorage
(
OpenCLRuntime
::
kPrecompiledBinaryPath
));
if
(
precompiled_binary_storage_
->
Load
()
!=
0
)
{
LOG
(
FATAL
)
<<
"Load OpenCL precompiled kernel file failed"
;
}
auto
platform_info_array
=
this
->
precompiled_binary_storage_
->
Find
(
kOpenCLPlatformInfoKey
);
if
(
platform_info_array
!=
nullptr
)
{
precompiled_binary_platform_info_
=
std
::
string
(
platform_info_array
->
begin
(),
platform_info_array
->
end
());
}
}
}
...
...
@@ -416,16 +478,23 @@ uint32_t OpenCLRuntime::device_compute_units() const {
return
device_compute_units_
;
}
bool
OpenCLRuntime
::
BuildProgramFrom
Binary
(
bool
OpenCLRuntime
::
BuildProgramFrom
Cache
(
const
std
::
string
&
built_program_key
,
const
std
::
string
&
build_options_str
,
cl
::
Program
*
program
)
{
// Find from binary
if
(
kCompiledProgramPlatform
!=
platform_info_
)
return
false
;
auto
it_binary
=
kCompiledProgramMap
.
find
(
built_program_key
);
if
(
it_binary
==
kCompiledProgramMap
.
end
())
return
false
;
if
(
this
->
cache_storage_
==
nullptr
)
return
false
;
if
(
cached_binary_platform_info_
!=
platform_info_
)
{
VLOG
(
3
)
<<
"cached OpenCL binary version is not same"
" with current version"
;
return
false
;
}
auto
content
=
this
->
cache_storage_
->
Find
(
built_program_key
);
if
(
content
==
nullptr
)
{
return
false
;
}
*
program
=
cl
::
Program
(
context
(),
{
device
()},
{
it_binary
->
second
});
*
program
=
cl
::
Program
(
context
(),
{
device
()},
{
*
content
});
cl_int
ret
=
program
->
build
({
device
()},
build_options_str
.
c_str
());
if
(
ret
!=
CL_SUCCESS
)
{
if
(
program
->
getBuildInfo
<
CL_PROGRAM_BUILD_STATUS
>
(
device
())
==
...
...
@@ -435,25 +504,27 @@ bool OpenCLRuntime::BuildProgramFromBinary(
LOG
(
INFO
)
<<
"Program build log: "
<<
build_log
;
}
LOG
(
WARNING
)
<<
"Build program "
<<
built_program_key
<<
" from Binary failed:"
<<
(
ret
==
CL_INVALID_PROGRAM
?
"CL_INVALID_PROGRAM, possible "
"cause 1: the MACE library is built from SoC 1 but is "
"used on different SoC 2, possible cause 2: the MACE "
"buffer is corrupted make sure your code has no "
"out-of-range memory writing"
:
MakeString
(
ret
));
<<
built_program_key
<<
" from Cache failed:"
<<
MakeString
(
ret
);
return
false
;
}
VLOG
(
3
)
<<
"Program from
Binary
: "
<<
built_program_key
;
VLOG
(
3
)
<<
"Program from
Cache
: "
<<
built_program_key
;
return
true
;
}
bool
OpenCLRuntime
::
BuildProgramFrom
Cache
(
bool
OpenCLRuntime
::
BuildProgramFrom
PrecompiledBinary
(
const
std
::
string
&
built_program_key
,
const
std
::
string
&
build_options_str
,
cl
::
Program
*
program
)
{
// Find from binary
if
(
this
->
storage_
==
nullptr
)
return
false
;
auto
content
=
this
->
storage_
->
Find
(
built_program_key
);
if
(
this
->
precompiled_binary_storage_
==
nullptr
)
return
false
;
if
(
precompiled_binary_platform_info_
!=
platform_info_
)
{
VLOG
(
3
)
<<
"precompiled OpenCL binary version "
<<
precompiled_binary_platform_info_
<<
" is not same with current version"
;
return
false
;
}
auto
content
=
this
->
precompiled_binary_storage_
->
Find
(
built_program_key
);
if
(
content
==
nullptr
)
{
return
false
;
}
...
...
@@ -468,11 +539,11 @@ bool OpenCLRuntime::BuildProgramFromCache(
LOG
(
INFO
)
<<
"Program build log: "
<<
build_log
;
}
LOG
(
WARNING
)
<<
"Build program "
<<
built_program_key
<<
" from
Cache
failed:"
<<
built_program_key
<<
" from
precompiled binary
failed:"
<<
MakeString
(
ret
);
return
false
;
}
VLOG
(
3
)
<<
"Program from
Cache
: "
<<
built_program_key
;
VLOG
(
3
)
<<
"Program from
precompiled binary
: "
<<
built_program_key
;
return
true
;
}
...
...
@@ -527,8 +598,8 @@ void OpenCLRuntime::BuildProgramFromSource(
reinterpret_cast
<
unsigned
char
const
*>
(
program_binaries
[
0
].
get
())
+
program_binary_sizes
[
0
]);
if
(
this
->
storage_
!=
nullptr
)
{
this
->
storage_
->
Insert
(
built_program_key
,
content
);
if
(
this
->
cache_
storage_
!=
nullptr
)
{
this
->
cache_
storage_
->
Insert
(
built_program_key
,
content
);
}
VLOG
(
3
)
<<
"Program from source: "
<<
built_program_key
;
...
...
@@ -543,13 +614,12 @@ void OpenCLRuntime::BuildProgram(const std::string &program_name,
std
::
string
build_options_str
=
build_options
+
" -Werror -cl-mad-enable -cl-fast-relaxed-math"
;
//
TODO(heliangliang) -cl-unsafe-math-optimizations -cl-fast-relaxed-math
bool
ret
=
BuildProgramFrom
Binary
(
built_program_key
,
build_options_str
,
program
);
//
Build flow: cache -> precompiled binary -> source
bool
ret
=
BuildProgramFrom
Cache
(
built_program_key
,
build_options_str
,
program
);
if
(
!
ret
)
{
ret
=
BuildProgramFromCache
(
built_program_key
,
build_options_str
,
program
);
// Fallback to source.
ret
=
BuildProgramFromPrecompiledBinary
(
built_program_key
,
build_options_str
,
program
);
if
(
!
ret
)
{
BuildProgramFromSource
(
program_name
,
built_program_key
,
build_options_str
,
program
);
...
...
@@ -581,8 +651,12 @@ cl::Kernel OpenCLRuntime::BuildKernel(
}
void
OpenCLRuntime
::
SaveBuiltCLProgram
()
{
if
(
storage_
!=
nullptr
)
{
if
(
storage_
->
Flush
()
!=
0
)
{
if
(
cache_storage_
!=
nullptr
)
{
// update platform info
cache_storage_
->
Insert
(
kOpenCLPlatformInfoKey
,
std
::
vector
<
unsigned
char
>
(
platform_info_
.
begin
(),
platform_info_
.
end
()));
if
(
cache_storage_
->
Flush
()
!=
0
)
{
LOG
(
FATAL
)
<<
"Store OPENCL compiled kernel to file failed."
" Please Make sure the storage directory exist."
;
}
...
...
mace/core/runtime/opencl/opencl_runtime.h
浏览文件 @
39fb3055
...
...
@@ -66,6 +66,7 @@ class OpenCLRuntime {
public:
static
OpenCLRuntime
*
Global
();
static
void
Configure
(
GPUPerfHint
,
GPUPriorityHint
);
static
void
ConfigureOpenCLBinaryPath
(
const
std
::
vector
<
std
::
string
>
&
paths
);
cl
::
Context
&
context
();
cl
::
Device
&
device
();
...
...
@@ -99,11 +100,11 @@ class OpenCLRuntime {
const
std
::
string
&
binary_file_name
,
const
std
::
string
&
build_options
,
cl
::
Program
*
program
);
bool
BuildProgramFrom
Binary
(
bool
BuildProgramFrom
Cache
(
const
std
::
string
&
built_program_key
,
const
std
::
string
&
build_options_str
,
cl
::
Program
*
program
);
bool
BuildProgramFrom
Cache
(
bool
BuildProgramFrom
PrecompiledBinary
(
const
std
::
string
&
built_program_key
,
const
std
::
string
&
build_options_str
,
cl
::
Program
*
program
);
...
...
@@ -115,7 +116,8 @@ class OpenCLRuntime {
const
std
::
string
ParseDeviceVersion
(
const
std
::
string
&
device_version
);
private:
std
::
unique_ptr
<
KVStorage
>
storage_
;
std
::
unique_ptr
<
KVStorage
>
precompiled_binary_storage_
;
std
::
unique_ptr
<
KVStorage
>
cache_storage_
;
bool
is_profiling_enabled_
;
// All OpenCL object must be a pointer and manually deleted before unloading
// OpenCL library.
...
...
@@ -126,6 +128,8 @@ class OpenCLRuntime {
std
::
mutex
program_build_mutex_
;
std
::
string
platform_info_
;
std
::
string
opencl_version_
;
std
::
string
precompiled_binary_platform_info_
;
std
::
string
cached_binary_platform_info_
;
bool
out_of_range_check_
;
uint64_t
device_gloabl_mem_cache_size_
;
uint32_t
device_compute_units_
;
...
...
@@ -133,6 +137,7 @@ class OpenCLRuntime {
static
GPUPerfHint
kGPUPerfHint
;
static
GPUPriorityHint
kGPUPriorityHint
;
static
std
::
string
kPrecompiledBinaryPath
;
};
}
// namespace mace
...
...
mace/examples/example.cc
浏览文件 @
39fb3055
...
...
@@ -123,6 +123,9 @@ DEFINE_string(model_data_file,
DEFINE_string
(
model_file
,
""
,
"model file name, used when load mace model in pb"
);
DEFINE_string
(
opencl_binary_file
,
""
,
"compiled opencl binary file path"
);
DEFINE_string
(
device
,
"GPU"
,
"CPU/GPU/HEXAGON"
);
DEFINE_int32
(
round
,
1
,
"round"
);
DEFINE_int32
(
restart_round
,
1
,
"restart round"
);
...
...
@@ -151,6 +154,10 @@ bool RunModel(const std::vector<std::string> &input_names,
}
#endif // MACE_ENABLE_OPENCL
if
(
device_type
==
DeviceType
::
GPU
)
{
std
::
vector
<
std
::
string
>
opencl_binary_paths
=
{
FLAGS_opencl_binary_file
};
mace
::
SetOpenCLBinaryPaths
(
opencl_binary_paths
);
}
// DO NOT USE tmp directory.
// Please use APP's own directory and make sure the directory exists.
// Just call once
...
...
mace/public/mace_runtime.h
浏览文件 @
39fb3055
...
...
@@ -76,9 +76,16 @@ class FileStorageFactory : public KVStorageFactory {
std
::
unique_ptr
<
Impl
>
impl_
;
};
// Set KV store factory used as OpenCL cache.
// Set KV store factory used as OpenCL cache.
(Call Once)
void
SetKVStorageFactory
(
std
::
shared_ptr
<
KVStorageFactory
>
storage_factory
);
// Just call once. (Not thread-safe)
// Set paths of OpenCL Compiled Binary file if you use gpu of specific soc.
// Using OpenCL binary will speed up the initialization.
// OpenCL binary is corresponding to the OpenCL Driver version,
// you should update the binary when OpenCL Driver changed.
void
SetOpenCLBinaryPaths
(
const
std
::
vector
<
std
::
string
>
&
paths
);
// Set GPU hints, currently only supports Adreno GPU.
//
// Caution: this function may hurt performance if improper parameters provided.
...
...
mace/python/tools/opencl_codegen.py
已删除
100644 → 0
浏览文件 @
d4124708
# Copyright 2018 Xiaomi, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
argparse
import
os
import
sys
import
struct
import
numpy
as
np
import
jinja2
# python mace/python/tools/opencl_codegen.py \
# --cl_binary_dirs=${CL_BIN_DIR} --output_path=${CL_HEADER_PATH}
FLAGS
=
None
def
generate_cpp_source
(
cl_binary_dirs
,
built_kernel_file_name
,
platform_info_file_name
):
maps
=
{}
platform_info
=
''
binary_dirs
=
cl_binary_dirs
.
strip
().
split
(
","
)
for
binary_dir
in
binary_dirs
:
binary_path
=
os
.
path
.
join
(
binary_dir
,
built_kernel_file_name
)
if
not
os
.
path
.
exists
(
binary_path
):
continue
print
'generate opencl code from'
,
binary_path
with
open
(
binary_path
,
"rb"
)
as
f
:
binary_array
=
np
.
fromfile
(
f
,
dtype
=
np
.
uint8
)
idx
=
0
size
,
=
struct
.
unpack
(
"Q"
,
binary_array
[
idx
:
idx
+
8
])
idx
+=
8
for
_
in
xrange
(
size
):
key_size
,
=
struct
.
unpack
(
"i"
,
binary_array
[
idx
:
idx
+
4
])
idx
+=
4
key
,
=
struct
.
unpack
(
str
(
key_size
)
+
"s"
,
binary_array
[
idx
:
idx
+
key_size
])
idx
+=
key_size
value_size
,
=
struct
.
unpack
(
"i"
,
binary_array
[
idx
:
idx
+
4
])
idx
+=
4
maps
[
key
]
=
[]
value
=
struct
.
unpack
(
str
(
value_size
)
+
"B"
,
binary_array
[
idx
:
idx
+
value_size
])
idx
+=
value_size
for
ele
in
value
:
maps
[
key
].
append
(
hex
(
ele
))
cl_platform_info_path
=
os
.
path
.
join
(
binary_dir
,
platform_info_file_name
)
with
open
(
cl_platform_info_path
,
'r'
)
as
f
:
curr_platform_info
=
f
.
read
()
if
platform_info
!=
""
:
assert
(
curr_platform_info
==
platform_info
)
platform_info
=
curr_platform_info
env
=
jinja2
.
Environment
(
loader
=
jinja2
.
FileSystemLoader
(
sys
.
path
[
0
]))
return
env
.
get_template
(
'opencl_compiled_kernel.cc.jinja2'
).
render
(
maps
=
maps
,
data_type
=
'unsigned char'
,
variable_name
=
'kCompiledProgramMap'
,
platform_info
=
platform_info
,
)
def
opencl_codegen
(
output_path
,
cl_binary_dirs
=
""
,
built_kernel_file_name
=
""
,
platform_info_file_name
=
""
):
cpp_cl_binary_source
=
generate_cpp_source
(
cl_binary_dirs
,
built_kernel_file_name
,
platform_info_file_name
)
if
os
.
path
.
isfile
(
output_path
):
os
.
remove
(
output_path
)
with
open
(
output_path
,
"w"
)
as
w_file
:
w_file
.
write
(
cpp_cl_binary_source
)
def
parse_args
():
"""Parses command line arguments."""
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
"--cl_binary_dirs"
,
type
=
str
,
default
=
""
,
help
=
"The cl binaries directories."
)
parser
.
add_argument
(
"--built_kernel_file_name"
,
type
=
str
,
default
=
""
,
help
=
"The cl binaries directories."
)
parser
.
add_argument
(
"--platform_info_file_name"
,
type
=
str
,
default
=
""
,
help
=
"The cl binaries directories."
)
parser
.
add_argument
(
"--output_path"
,
type
=
str
,
default
=
"./mace/examples/codegen/opencl/opencl_compiled_program.cc"
,
help
=
"The path of generated C++ header file for cl binaries."
)
return
parser
.
parse_known_args
()
if
__name__
==
'__main__'
:
FLAGS
,
unparsed
=
parse_args
()
opencl_codegen
(
FLAGS
.
output_path
,
FLAGS
.
cl_binary_dirs
,
FLAGS
.
built_kernel_file_name
,
FLAGS
.
platform_info_file_name
)
mace/python/tools/opencl_compiled_kernel.cc.jinja2
已删除
100644 → 0
浏览文件 @
d4124708
// Copyright 2018 Xiaomi, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This is a generated file. DO NOT EDIT!
#include <map>
#include <string>
#include <vector>
namespace mace {
extern const std::map<std::string, std::vector<{{data_type}}>> {{variable_name}} =
{
{% for key, value in maps.iteritems() %}
{
"{{key}}",
{
{%- for ele in value -%}
{{ele}},
{%- endfor -%}
}
}, // {{key}}
{% endfor %}
};
extern const std::string kCompiledProgramPlatform = {{platform_info|tojson}};
} // namespace mace
mace/tools/validation/BUILD
浏览文件 @
39fb3055
...
...
@@ -13,6 +13,5 @@ cc_binary(
"//external:gflags_nothreads"
,
"//mace/codegen:generated_mace_engine_factory"
,
"//mace/codegen:generated_models"
,
"//mace/core"
,
],
)
mace/tools/validation/mace_run.cc
浏览文件 @
39fb3055
...
...
@@ -38,9 +38,6 @@
#include "mace/utils/logging.h"
#include "mace/utils/utils.h"
#ifdef MACE_ENABLE_OPENCL
#include "mace/core/runtime/opencl/opencl_runtime.h"
#endif // MACE_ENABLE_OPENCL
#include "mace/codegen/engine/mace_engine_factory.h"
namespace
mace
{
...
...
@@ -100,22 +97,6 @@ DeviceType ParseDeviceType(const std::string &device_str) {
}
}
#ifdef MACE_ENABLE_OPENCL
void
WriteOpenCLPlatformInfo
(
const
std
::
string
&
output_dir
)
{
std
::
string
platform_info
=
OpenCLRuntime
::
Global
()
->
platform_info
();
const
std
::
string
cl_platform_info_file_name
=
output_dir
+
"/mace_cl_platform_info.txt"
;
std
::
ofstream
ofs
(
cl_platform_info_file_name
);
if
(
ofs
.
is_open
())
{
ofs
<<
platform_info
;
ofs
.
close
();
}
else
{
LOG
(
WARNING
)
<<
"Write opencl platform info failed."
;
}
}
#endif // MACE_ENABLE_OPENCL
struct
mallinfo
LogMallinfoChange
(
struct
mallinfo
prev
)
{
struct
mallinfo
curr
=
mallinfo
();
if
(
prev
.
arena
!=
curr
.
arena
)
{
...
...
@@ -187,6 +168,9 @@ DEFINE_string(input_file,
DEFINE_string
(
output_file
,
""
,
"output file name | output file prefix for multiple outputs"
);
DEFINE_string
(
opencl_binary_file
,
""
,
"compiled opencl binary file path"
);
DEFINE_string
(
model_data_file
,
""
,
"model data file name, used when EMBED_MODEL_DATA set to 0 or 2"
);
...
...
@@ -230,6 +214,11 @@ bool RunModel(const std::string &model_name,
new
FileStorageFactory
(
kernel_file_path
));
SetKVStorageFactory
(
storage_factory
);
if
(
device_type
==
DeviceType
::
GPU
)
{
std
::
vector
<
std
::
string
>
opencl_binary_paths
=
{
FLAGS_opencl_binary_file
};
mace
::
SetOpenCLBinaryPaths
(
opencl_binary_paths
);
}
std
::
vector
<
unsigned
char
>
model_pb_data
;
if
(
FLAGS_model_file
!=
""
)
{
if
(
!
mace
::
ReadBinaryFile
(
&
model_pb_data
,
FLAGS_model_file
))
{
...
...
@@ -397,11 +386,6 @@ bool RunModel(const std::string &model_name,
printf
(
"time %11.3f %11.3f %11.3f
\n
"
,
init_millis
,
warmup_millis
,
model_run_millis
);
#ifdef MACE_ENABLE_OPENCL
if
(
device_type
==
DeviceType
::
GPU
)
{
WriteOpenCLPlatformInfo
(
kernel_file_path
);
}
#endif // MACE_ENABLE_OPENCL
for
(
size_t
i
=
0
;
i
<
output_count
;
++
i
)
{
std
::
string
output_name
=
...
...
tools/bazel_adb_run.py
浏览文件 @
39fb3055
...
...
@@ -132,7 +132,6 @@ def main(unused_args):
# generate sources
sh_commands
.
gen_encrypted_opencl_source
()
sh_commands
.
gen_compiled_opencl_source
()
sh_commands
.
gen_mace_version
()
sh_commands
.
gen_tuning_param_code
([])
...
...
tools/common.py
浏览文件 @
39fb3055
...
...
@@ -108,6 +108,15 @@ class StringFormatter:
return
star_line
+
str
(
message
).
center
(
line_length
)
+
'
\n
'
+
star_line
################################
# definitions
################################
class
DeviceType
(
object
):
CPU
=
'CPU'
GPU
=
'GPU'
HEXAGON
=
'HEXAGON'
################################
# Argument types
################################
...
...
tools/converter.py
浏览文件 @
39fb3055
...
...
@@ -29,6 +29,7 @@ import sh_commands
from
sh_commands
import
BuildType
from
common
import
CaffeEnvType
from
common
import
DeviceType
from
common
import
mace_check
from
common
import
MaceLogger
from
common
import
StringFormatter
...
...
@@ -37,13 +38,14 @@ from common import StringFormatter
# common definitions
################################
BUILD_OUTPUT_DIR
=
'build'
PHONE_DATA_DIR
=
"/data/local/tmp/mace_run
/
"
PHONE_DATA_DIR
=
"/data/local/tmp/mace_run"
MODEL_OUTPUT_DIR_NAME
=
'model'
BUILD_TMP_DIR_NAME
=
'_tmp'
BUILD_TMP_GENERAL_OUTPUT_DIR_NAME
=
'general'
OUTPUT_LIBRARY_DIR_NAME
=
'library'
CL_BUILT_KERNEL_FILE_NAME
=
"mace_cl_compiled_program.bin"
CL_PLATFORM_INFO_FILE_NAME
=
"mace_cl_platform_info.txt"
OUTPUT_OPENCL_BINARY_DIR_NAME
=
'opencl'
OUTPUT_OPENCL_BINARY_FILE_NAME
=
'compiled_opencl_kernel.bin'
CL_COMPILED_BINARY_FILE_NAME
=
"mace_cl_compiled_program.bin"
CODEGEN_BASE_DIR
=
'mace/codegen'
MODEL_CODEGEN_DIR
=
CODEGEN_BASE_DIR
+
'/models'
MACE_RUN_TARGET
=
"//mace/tools/validation:mace_run"
...
...
@@ -176,11 +178,11 @@ def parse_device_type(runtime):
device_type
=
""
if
runtime
==
RuntimeType
.
dsp
:
device_type
=
"HEXAGON"
device_type
=
DeviceType
.
HEXAGON
elif
runtime
==
RuntimeType
.
gpu
:
device_type
=
"GPU"
device_type
=
DeviceType
.
GPU
elif
runtime
==
RuntimeType
.
cpu
:
device_type
=
"CPU"
device_type
=
DeviceType
.
CPU
return
device_type
...
...
@@ -433,6 +435,13 @@ def get_build_model_dirs(library_name, model_name, target_abi, target_soc,
return
model_output_base_dir
,
model_output_dir
,
mace_model_dir
def
get_opencl_binary_output_path
(
library_name
):
return
'%s/%s/%s/%s'
%
(
BUILD_OUTPUT_DIR
,
library_name
,
OUTPUT_OPENCL_BINARY_DIR_NAME
,
OUTPUT_OPENCL_BINARY_FILE_NAME
)
################################
# build
################################
...
...
@@ -440,17 +449,7 @@ def pull_opencl_binary_and_tuning_param(target_abi,
serialno
,
model_output_dirs
):
sh_commands
.
pull_binaries
(
target_abi
,
serialno
,
model_output_dirs
,
CL_BUILT_KERNEL_FILE_NAME
,
CL_PLATFORM_INFO_FILE_NAME
)
def
gen_opencl_and_tuning_code
(
model_output_dirs
):
# generate opencl binary code
sh_commands
.
gen_opencl_binary_code
(
model_output_dirs
,
CL_BUILT_KERNEL_FILE_NAME
,
CL_PLATFORM_INFO_FILE_NAME
)
sh_commands
.
gen_tuning_param_code
(
model_output_dirs
)
CL_COMPILED_BINARY_FILE_NAME
)
def
print_configuration
(
flags
,
configs
):
...
...
@@ -612,7 +611,7 @@ def build_specific_lib(target_abi, target_soc, serial_num,
sh
.
rm
(
"-rf"
,
build_tmp_binary_dir
)
os
.
makedirs
(
build_tmp_binary_dir
)
gen_opencl_and_tuning_code
([]
)
sh_commands
.
gen_tuning_param_code
(
model_output_dirs
)
sh_commands
.
bazel_build
(
MACE_RUN_TARGET
,
abi
=
target_abi
,
...
...
@@ -639,7 +638,7 @@ def build_specific_lib(target_abi, target_soc, serial_num,
os
.
makedirs
(
model_output_dir
)
# build for specified soc
if
not
address_sanitizer
and
t
uning
and
t
arget_abi
!=
ABIType
.
host
\
if
not
address_sanitizer
and
target_abi
!=
ABIType
.
host
\
and
target_soc
is
not
None
and
\
model_runtime
in
[
RuntimeType
.
gpu
,
RuntimeType
.
cpu_gpu
]:
sh_commands
.
clear_phone_data_dir
(
serial_num
,
PHONE_DATA_DIR
)
...
...
@@ -674,7 +673,8 @@ def build_specific_lib(target_abi, target_soc, serial_num,
tuning
=
tuning
,
out_of_range_check
=
False
,
phone_data_dir
=
PHONE_DATA_DIR
,
build_type
=
build_type
build_type
=
build_type
,
opencl_binary_file
=
""
,
)
pull_opencl_binary_and_tuning_param
(
target_abi
,
serial_num
,
...
...
@@ -683,7 +683,10 @@ def build_specific_lib(target_abi, target_soc, serial_num,
binary_changed
=
True
if
binary_changed
:
gen_opencl_and_tuning_code
(
model_output_dirs
)
sh_commands
.
merge_opencl_binaries
(
model_output_dirs
,
CL_COMPILED_BINARY_FILE_NAME
,
get_opencl_binary_output_path
(
library_name
))
sh_commands
.
gen_tuning_param_code
(
model_output_dirs
)
sh_commands
.
bazel_build
(
MACE_RUN_TARGET
,
abi
=
target_abi
,
...
...
@@ -919,6 +922,7 @@ def run_specific_target(flags, configs, target_abi,
gpu_priority_hint
=
flags
.
gpu_priority_hint
,
runtime_failure_ratio
=
flags
.
runtime_failure_ratio
,
address_sanitizer
=
flags
.
address_sanitizer
,
opencl_binary_file
=
get_opencl_binary_output_path
(
library_name
),
)
if
flags
.
validate
:
model_file_path
,
weight_file_path
=
get_model_files_path
(
...
...
@@ -1051,7 +1055,8 @@ def bm_specific_target(flags, configs, target_abi, target_soc, serial_num):
omp_num_threads
=
flags
.
omp_num_threads
,
cpu_affinity_policy
=
flags
.
cpu_affinity_policy
,
gpu_perf_hint
=
flags
.
gpu_perf_hint
,
gpu_priority_hint
=
flags
.
gpu_priority_hint
)
gpu_priority_hint
=
flags
.
gpu_priority_hint
,
opencl_binary_file
=
get_opencl_binary_output_path
(
library_name
))
def
benchmark_model
(
flags
):
...
...
tools/sh_commands.py
浏览文件 @
39fb3055
...
...
@@ -16,9 +16,11 @@ import falcon_cli
import
filelock
import
glob
import
logging
import
numpy
as
np
import
os
import
re
import
sh
import
struct
import
subprocess
import
sys
import
time
...
...
@@ -30,7 +32,6 @@ import common
sys
.
path
.
insert
(
0
,
"mace/python/tools"
)
try
:
from
encrypt_opencl_codegen
import
encrypt_opencl_codegen
from
opencl_codegen
import
opencl_codegen
from
binary_codegen
import
tuning_param_codegen
from
generate_data
import
generate_input_data
from
validate
import
validate
...
...
@@ -362,8 +363,7 @@ def gen_mace_engine_factory_source(model_tags,
def
pull_binaries
(
abi
,
serialno
,
model_output_dirs
,
cl_built_kernel_file_name
,
cl_platform_info_file_name
):
cl_built_kernel_file_name
):
compiled_opencl_dir
=
"/data/local/tmp/mace_run/interior/"
mace_run_param_file
=
"mace_run.config"
...
...
@@ -379,26 +379,66 @@ def pull_binaries(abi, serialno, model_output_dirs,
if
abi
!=
"host"
:
adb_pull
(
compiled_opencl_dir
+
cl_built_kernel_file_name
,
cl_bin_dir
,
serialno
)
adb_pull
(
compiled_opencl_dir
+
cl_platform_info_file_name
,
cl_bin_dir
,
serialno
)
adb_pull
(
"/data/local/tmp/mace_run/%s"
%
mace_run_param_file
,
cl_bin_dir
,
serialno
)
def
gen_opencl_binary_code
(
model_output_dirs
,
cl_built_kernel_file_name
,
cl_platform_info_file_name
,
codegen_path
=
"mace/codegen"
):
opencl_codegen_file
=
"%s/opencl/opencl_compiled_program.cc"
%
codegen_path
def
merge_opencl_binaries
(
binaries_dirs
,
cl_compiled_program_file_name
,
output_file_path
):
platform_info_key
=
'mace_opencl_precompiled_platform_info_key'
cl_bin_dirs
=
[]
for
d
in
model_output
_dirs
:
for
d
in
binaries
_dirs
:
cl_bin_dirs
.
append
(
os
.
path
.
join
(
d
,
"opencl_bin"
))
cl_bin_dirs_str
=
","
.
join
(
cl_bin_dirs
)
opencl_codegen
(
opencl_codegen_file
,
cl_bin_dirs_str
,
cl_built_kernel_file_name
,
cl_platform_info_file_name
)
# create opencl binary output dir
opencl_binary_dir
=
os
.
path
.
dirname
(
output_file_path
)
if
os
.
path
.
exists
(
opencl_binary_dir
):
sh
.
rm
(
"-rf"
,
opencl_binary_dir
)
sh
.
mkdir
(
"-p"
,
opencl_binary_dir
)
kvs
=
{}
for
binary_dir
in
cl_bin_dirs
:
binary_path
=
os
.
path
.
join
(
binary_dir
,
cl_compiled_program_file_name
)
if
not
os
.
path
.
exists
(
binary_path
):
continue
print
'generate opencl code from'
,
binary_path
with
open
(
binary_path
,
"rb"
)
as
f
:
binary_array
=
np
.
fromfile
(
f
,
dtype
=
np
.
uint8
)
idx
=
0
size
,
=
struct
.
unpack
(
"Q"
,
binary_array
[
idx
:
idx
+
8
])
idx
+=
8
for
_
in
xrange
(
size
):
key_size
,
=
struct
.
unpack
(
"i"
,
binary_array
[
idx
:
idx
+
4
])
idx
+=
4
key
,
=
struct
.
unpack
(
str
(
key_size
)
+
"s"
,
binary_array
[
idx
:
idx
+
key_size
])
idx
+=
key_size
value_size
,
=
struct
.
unpack
(
"i"
,
binary_array
[
idx
:
idx
+
4
])
idx
+=
4
if
key
==
platform_info_key
and
key
in
kvs
:
common
.
mace_check
(
(
kvs
[
key
]
==
binary_array
[
idx
:
idx
+
value_size
]).
all
(),
""
,
"There exists more than one OpenCL version for models:"
" %s vs %s "
%
(
kvs
[
key
],
binary_array
[
idx
:
idx
+
value_size
]))
else
:
kvs
[
key
]
=
binary_array
[
idx
:
idx
+
value_size
]
idx
+=
value_size
output_byte_array
=
bytearray
()
data_size
=
len
(
kvs
)
output_byte_array
.
extend
(
struct
.
pack
(
"Q"
,
data_size
))
for
key
,
value
in
kvs
.
iteritems
():
key_size
=
len
(
key
)
output_byte_array
.
extend
(
struct
.
pack
(
"i"
,
key_size
))
output_byte_array
.
extend
(
struct
.
pack
(
str
(
key_size
)
+
"s"
,
key
))
value_size
=
len
(
value
)
output_byte_array
.
extend
(
struct
.
pack
(
"i"
,
value_size
))
output_byte_array
.
extend
(
value
)
np
.
array
(
output_byte_array
).
tofile
(
output_file_path
)
def
gen_tuning_param_code
(
model_output_dirs
,
...
...
@@ -426,12 +466,6 @@ def gen_mace_version(codegen_path="mace/codegen"):
"%s/version/version.cc"
%
codegen_path
)
def
gen_compiled_opencl_source
(
codegen_path
=
"mace/codegen"
):
opencl_codegen_file
=
"%s/opencl/opencl_compiled_program.cc"
%
codegen_path
sh
.
mkdir
(
"-p"
,
"%s/opencl"
%
codegen_path
)
opencl_codegen
(
opencl_codegen_file
)
def
gen_model_code
(
model_codegen_dir
,
platform
,
model_file_path
,
...
...
@@ -576,6 +610,7 @@ def tuning_run(abi,
out_of_range_check
,
phone_data_dir
,
build_type
,
opencl_binary_file
,
omp_num_threads
=-
1
,
cpu_affinity_policy
=
1
,
gpu_perf_hint
=
3
,
...
...
@@ -641,6 +676,10 @@ def tuning_run(abi,
adb_push
(
"%s/%s.data"
%
(
mace_model_dir
,
model_tag
),
phone_data_dir
,
serialno
)
if
device_type
==
common
.
DeviceType
.
GPU
\
and
os
.
path
.
exists
(
opencl_binary_file
):
adb_push
(
opencl_binary_file
,
phone_data_dir
,
serialno
)
adb_push
(
"third_party/nnlib/libhexagon_controller.so"
,
phone_data_dir
,
serialno
)
...
...
@@ -689,6 +728,8 @@ def tuning_run(abi,
"--gpu_perf_hint=%s"
%
gpu_perf_hint
,
"--gpu_priority_hint=%s"
%
gpu_priority_hint
,
"--model_file=%s"
%
mace_model_phone_path
,
"--opencl_binary_file=%s/%s"
%
(
phone_data_dir
,
os
.
path
.
basename
(
opencl_binary_file
)),
])
adb_cmd
=
' '
.
join
(
adb_cmd
)
sh
.
adb
(
...
...
@@ -1005,6 +1046,7 @@ def benchmark_model(abi,
device_type
,
phone_data_dir
,
build_type
,
opencl_binary_file
,
omp_num_threads
=-
1
,
cpu_affinity_policy
=
1
,
gpu_perf_hint
=
3
,
...
...
@@ -1049,6 +1091,9 @@ def benchmark_model(abi,
if
not
embed_model_data
:
adb_push
(
"%s/%s.data"
%
(
mace_model_dir
,
model_tag
),
phone_data_dir
,
serialno
)
if
device_type
==
common
.
DeviceType
.
GPU
\
and
os
.
path
.
exists
(
opencl_binary_file
):
adb_push
(
opencl_binary_file
,
phone_data_dir
,
serialno
)
mace_model_phone_path
=
""
if
build_type
==
BuildType
.
proto
:
mace_model_phone_path
=
"%s/%s.pb"
%
(
phone_data_dir
,
model_tag
)
...
...
@@ -1082,6 +1127,8 @@ def benchmark_model(abi,
"--gpu_perf_hint=%s"
%
gpu_perf_hint
,
"--gpu_priority_hint=%s"
%
gpu_priority_hint
,
"--model_file=%s"
%
mace_model_phone_path
,
"--opencl_binary_file=%s/%s"
%
(
phone_data_dir
,
os
.
path
.
basename
(
opencl_binary_file
)),
_fg
=
True
)
print
(
"Benchmark done!
\n
"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录